---

import builtins
import json
import os
import sys
import logging
import traceback
import requests
import uuid
from datetime import datetime, timedelta
from calendar import monthrange
from io import StringIO
import csv
import time
from lxml.builder import E
from lxml import etree

from umdap_config import LOG_FOLDER, ENV_FOLDER, EROUTER_HOST, PRIMARY_MADAB, UMDAP_ETL_URL_LIST

APPLICATION_NAME = 'externalGTT'
LOG_FILE = os.path.join(LOG_FOLDER, "externalGTT_test.log")
CREDENTIALS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.json")

BASE_URL = 'https://www.globaltradetracker.com/api/rest'
builtins.BASE_URL = BASE_URL
builtins.XML_DATE_FMT = '%d.%m.%Y %H:%M:%S'
builtins.DELIMITER = ','

# === НАСТРОЙКИ ===
TARGET_ROWS = 100000
OUTPUT_XML = 'test.xml'
CSV_TEMP = 'test_batch.csv'

session = requests.Session()
session.headers.update({
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'application/json, text/plain, */*',
    'Accept-Language': 'en-US,en;q=0.9',
    'Accept-Encoding': 'gzip, deflate, br',
    'Connection': 'keep-alive',
})


def log(msg=''):
    str_with_time = f'{datetime.now().strftime("%Y/%m/%d %H:%M:%S")} {msg}'
    print(str_with_time)
    with open(LOG_FILE, "a", encoding="utf-8") as file:
        file.write(f'{str_with_time}\n')

builtins.log = log


def generate_id():
    return uuid.uuid4().hex


def get_token():
    try:
        with open(CREDENTIALS_FILE, 'r') as f:
            credentials = json.load(f)
        link = f"{BASE_URL}/gettoken?userid={credentials['user_id']}&password={credentials['password']}"
        response = session.get(link, timeout=30)
        if response.status_code == 200:
            token = response.content.decode().strip()
            if token:
                log("Token received")
                return token
        return None
    except Exception as e:
        log(f"ERROR get_token: {e}")
        return None


def get_subscription_countries(token):
    try:
        response = session.get(f"{BASE_URL}/countries?token={token}", timeout=30)
        if response.status_code == 200:
            data = response.json()
            countries = []
            if isinstance(data, list):
                for item in data:
                    if isinstance(item, dict):
                        country_data = item.get('country', item)
                        if country_data.get('reportingcountry') == 'true' and country_data.get('defaultReporter') == True:
                            code = country_data.get('reportercode', '')
                            if code:
                                countries.append(code)
            countries = list(dict.fromkeys(countries))
            log(f"Available reporters: {len(countries)}")
            return countries
        return []
    except Exception as e:
        log(f"ERROR get_countries: {e}")
        return []


def parse_period(year, month):
    try:
        year = int(year)
        month = int(month)
        last_day = monthrange(year, month)[1]
        return datetime(year, month, last_day).strftime('%Y-%m-%d')
    except:
        return '2000-01-01'


def parse_record_to_dict(record, reporter_code="", trade_flow=""):
    try:
        if not isinstance(record, dict):
            return None
        year = record.get('year', 2000)
        month = record.get('month', 1)
        return {
            'ID': generate_id(),
            'Reporter_Code': record.get('reporterCode', reporter_code),
            'Trade_Flow': record.get('tradeFlow', trade_flow),
            'Is_Mirror_Data': record.get('isMirror', ''),
            'Period': parse_period(year, month),
            'Reporter_Name': record.get('reporterName', ''),
            'Reporter_Description': record.get('reporterDescription', '')[:120],
            'Reporter_Source': record.get('reporterSource', '')[:120],
            'Incoterm': record.get('incoterm', '')[:60],
            'Partner_Code': record.get('partnerCode', '')[:60],
            'Partner_Name': record.get('partnerName', '')[:120],
            'Commodity_HS_Code': record.get('hsCode', '')[:60],
            'Commodity_Description': record.get('commodityDescriptionOriginal', '')[:250],
            'HS6_Code': record.get('hs6Code', '')[:60],
            'HS6_Code_Description': record.get('hs6DescriptionOriginal', '')[:250],
            'Subdivision': record.get('subdivision', '')[:120],
            'Port': record.get('port', '')[:120],
            'Transport': record.get('transport', '')[:120],
            'Foreign_Port': record.get('foreignPort', '')[:120],
            'US_State': record.get('usState', '')[:120],
            'Customs_Regime': record.get('customsRegime', '')[:120],
            'Suppression': record.get('suppression', '')[:3900],
            'Monetary_Value': record.get('value', 0),
            'Currency': record.get('currency', 'USD')[:60],
            'Secondary_Monetary_Value': record.get('value2', 0),
            'Secondary_Currency': record.get('currency2', '')[:60],
            'Primary_Quantity': record.get('quantity1', 0),
            'Primary_Quantity_Unit': record.get('unit1', '')[:60],
            'Secondary_Quantity': record.get('quantity2', 0),
            'Secondary_Quantity_Unit': record.get('unit2', '')[:60],
            'Primary_Quantity_Price': record.get('price1', 0),
            'Primary_Quantity_Price_Unit': record.get('priceunit1', '')[:60],
            'Secondary_Quantity_Price': record.get('price2', 0),
            'Secondary_Quantity_Price_Unit': record.get('priceunit2', '')[:60],
        }
    except Exception as e:
        log(f"ERROR parse: {e}")
        return None


def parse_data_to_rows(data, reporter_code="", trade_flow=""):
    rows = []
    if isinstance(data, list):
        records = data
    elif isinstance(data, dict):
        if 'reportData' in data:
            records = data['reportData']
        elif 'data' in data:
            records = data['data']
        else:
            return rows
    else:
        return rows

    for record in records:
        if isinstance(record, dict):
            parsed = parse_record_to_dict(record, reporter_code, trade_flow)
            if parsed:
                rows.append(parsed)
    return rows


def save_csv(rows, filepath):
    if not rows:
        return
    with open(filepath, 'w', newline='', encoding='utf-8') as f:
        writer = csv.DictWriter(f, fieldnames=rows[0].keys())
        writer.writeheader()
        writer.writerows(rows)
    log(f"CSV saved: {filepath} ({len(rows)} rows)")


def generate_xml(rows):
    """Генерация XML как etl_common.py"""
    xml_rows = []
    for row in rows:
        row_elems = []
        for key, value in row.items():
            tag = str(key).replace(' ', '_')
            if value is None:
                value = ''
            row_elems.append(E(tag, str(value)))
        xml_rows.append(E.row(*row_elems))

    root = E.madabContent(
        E.type('GTT'),
        *xml_rows
    )
    return etree.tostring(root, encoding='utf-8', pretty_print=True)


def get_historical_data(token, countries, target_rows=100000):
    log("=" * 60)
    log(f"Starting test parsing: {target_rows} rows")
    log("=" * 60)

    all_rows = []
    total_requests = 0
    hs_filter = '31,25'

    fields = 'reporterCode,tradeFlow,isMirror,year,month,reporterName,reporterDescription,reporterSource,incoterm,partnerCode,partnerName,hsCode,commodityDescriptionOriginal,hs6Code,hs6DescriptionOriginal,value,currency,value2,currency2,quantity1,unit1,quantity2,unit2,price1,priceunit1,price2,priceunit2,port,transport,subdivision'

    trade_details = 'PORT,SUBDIVISION,TRANSPORT,CUSTOMS_REGIME,FOREIGN_PORT,FOREIGN_SUBDIVISION'

    for country in countries:
        if len(all_rows) >= target_rows:
            log(f"Reached target rows: {len(all_rows)}")
            break

        for impexp in ['E', 'I']:
            if len(all_rows) >= target_rows:
                break

            try:
                params = {
                    'token': token,
                    'hscode': hs_filter,
                    'reporter': country,
                    'impexp': impexp,
                    'from': '2013-01',
                    'to': '2026-08',
                    'layout': 'brief',
                    'hslevel': '6',
                    'field': fields,
                    'tradedetails': trade_details,
                    'currency2': 'EUR',
                    'decimalscale': '2',
                    'format': 'json'
                }

                total_requests += 1

                start = time.time()
                response = session.get(f"{BASE_URL}/getreport", params=params, timeout=600)
                end = time.time()

                log(f"[{total_requests}] {country} {impexp} Status={response.status_code} Time={end-start:.1f}s Total rows={len(all_rows)}")

                if response.status_code == 200:
                    data = response.json()
                    new_rows = parse_data_to_rows(data, country, impexp)

                    remaining = target_rows - len(all_rows)
                    if len(new_rows) > remaining:
                        new_rows = new_rows[:remaining]

                    all_rows.extend(new_rows)
                    log(f"  Added {len(new_rows)} rows, total: {len(all_rows)}")

            except Exception as e:
                log(f"✗ {str(e)}")
                continue

    return all_rows


def main():
    log('--------------------- TEST SCRIPT started ------------------------------')

    try:
        token = get_token()
        if not token:
            log("ERROR: Failed to get token")
            sys.exit(1)

        countries = get_subscription_countries(token)
        if not countries:
            log("ERROR: Failed to get countries")
            sys.exit(1)

        log(f"Parsing {TARGET_ROWS} rows...")
        rows = get_historical_data(token, countries, TARGET_ROWS)

        if not rows:
            log("ERROR: No rows parsed")
            sys.exit(1)

        log(f"Parsed {len(rows)} rows")

        save_csv(rows, CSV_TEMP)

        log("Generating XML...")
        xml_bytes = generate_xml(rows)

        with open(OUTPUT_XML, 'wb') as f:
            f.write(xml_bytes)

        size_mb = len(xml_bytes) / 1024 / 1024
        log(f"XML saved: {OUTPUT_XML}")
        log(f"XML size: {size_mb:.2f} MB")
        log(f"Rows in XML: {len(rows)}")

        log('--------------------- TEST SCRIPT finished -----------------------------')
        log(f"Try: curl -X POST 'https://e-router.eurochem.ru/procedure/WHDEV/umdap.etl.content/call?application=externalGTT' --data-binary @{OUTPUT_XML} -k -v")

    except Exception as e:
        log(f'Error: {traceback.format_exception(*sys.exc_info())}')
        sys.exit(1)


if __name__ == "__main__":
    main()