INIT
This commit is contained in:
commit
1ea3de74f2
10 files changed
+1825
No files matched your search
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
import csv
|
||||
import sys
|
||||
import time
|
||||
import logging
|
||||
import argparse
|
||||
import requests
|
||||
|
||||
WEBHOOK_URL = "https://brandize.app.n8n.cloud/webhook/VAT"
|
||||
BATCH_SIZE = 50
|
||||
REQUEST_TIMEOUT = 60
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)-7s %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
log = logging.getLogger("send")
|
||||
|
||||
|
||||
def load_rows(csv_file):
|
||||
with open(csv_file, "r", newline="", encoding="utf-8-sig") as file:
|
||||
return list(csv.DictReader(file))
|
||||
|
||||
|
||||
def send_batch(batch):
|
||||
response = requests.post(
|
||||
WEBHOOK_URL,
|
||||
json=batch,
|
||||
headers={"Content-Type": "application/json"},
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
||||
|
||||
def send_csv(csv_file, batch_size):
|
||||
try:
|
||||
rows = load_rows(csv_file)
|
||||
except FileNotFoundError:
|
||||
log.error("CSV file not found: %s", csv_file)
|
||||
return 1
|
||||
except OSError as e:
|
||||
log.error("Could not read %s: %s", csv_file, e)
|
||||
return 1
|
||||
|
||||
total = len(rows)
|
||||
if total == 0:
|
||||
log.warning("No data rows found in %s", csv_file)
|
||||
return 0
|
||||
|
||||
num_batches = (total + batch_size - 1) // batch_size
|
||||
log.info("Loaded %d rows from %s", total, csv_file)
|
||||
log.info("Sending in %d batch(es) of up to %d rows", num_batches, batch_size)
|
||||
|
||||
sent = 0
|
||||
failed = 0
|
||||
start_time = time.time()
|
||||
|
||||
for index in range(num_batches):
|
||||
batch = rows[index * batch_size:(index + 1) * batch_size]
|
||||
first_row = index * batch_size + 1
|
||||
last_row = first_row + len(batch) - 1
|
||||
|
||||
log.info(
|
||||
"Batch %d/%d (rows %d-%d, %d rows) ...",
|
||||
index + 1, num_batches, first_row, last_row, len(batch),
|
||||
)
|
||||
|
||||
try:
|
||||
response = send_batch(batch)
|
||||
sent += len(batch)
|
||||
log.info(
|
||||
"Batch %d/%d OK (HTTP %d) | Progress: %d/%d rows (%.2f%%)",
|
||||
index + 1, num_batches, response.status_code,
|
||||
sent, total, sent / total * 100,
|
||||
)
|
||||
except requests.RequestException as e:
|
||||
failed += 1
|
||||
log.error(
|
||||
"Batch %d/%d FAILED: %s | Progress: %d/%d rows (%.2f%%)",
|
||||
index + 1, num_batches, e,
|
||||
sent, total, sent / total * 100,
|
||||
)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
log.info(
|
||||
"Done in %.1fs. %d/%d rows sent, %d batch(es) failed. Progress: %.2f%%",
|
||||
elapsed, sent, total, failed, sent / total * 100,
|
||||
)
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Send all rows of a CSV file to an n8n webhook in batches."
|
||||
)
|
||||
parser.add_argument("csv_file", help="Path to the CSV file")
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default=BATCH_SIZE,
|
||||
help="Number of rows per request (default: %(default)s)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.batch_size < 1:
|
||||
parser.error("--batch-size must be at least 1.")
|
||||
|
||||
sys.exit(send_csv(args.csv_file, args.batch_size))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user