---

import builtins
import json
import os
import sys
import logging
import traceback
import requests
import uuid
from datetime import datetime, timedelta
from calendar import monthrange
from io import StringIO
import csv
import time
import gc
import ctypes
from lxml.builder import E
from lxml import etree

from umdap_config import LOG_FOLDER, ENV_FOLDER, EROUTER_HOST, PRIMARY_MADAB, UMDAP_ETL_URL_LIST

APPLICATION_NAME = 'externalGTT'
LOG_FILE = os.path.join(LOG_FOLDER, "externalGTT_test.log")

BASE_URL = 'https://www.globaltradetracker.com/api/rest'
builtins.BASE_URL = BASE_URL
builtins.XML_DATE_FMT = '%d.%m.%Y %H:%M:%S'
builtins.DELIMITER = ','

# === НАСТРОЙКИ ===
TARGET_ROWS = 100000          # сколько строк спарсить
OUTPUT_XML = 'test.xml'       # куда сохранить XML
CSV_TEMP = 'test_batch.csv'   # временный CSV

session = requests.Session()
session.headers.update({
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'application/json, text/plain, */*',
    'Accept-Language': 'en-US,en;q=0.9',
    'Accept-Encoding': 'gzip, deflate, br',
    'Connection': 'keep-alive',
})

# ID приложения
APPLICATION_NAME = 'externalGTT'


def log(msg=''):
    """Логирование"""
    str_with_time = f'{datetime.now().strftime("%Y/%m/%d %H:%M:%S")} {msg}'
    print(str_with_time)
    with open(LOG_FILE, "a", encoding="utf-8") as file:
        file.write(f'{str_with_time}\n')

builtins.log = log


def generate_id():
    """Генерация 32-символьного ID"""
    return uuid.uuid4().hex


def get_token():
    """Получение токена GTT"""
    try:
        with open('config.json', 'r') as f:
            credentials = json.load(f)
        link = f"{BASE_URL}/gettoken?userid={credentials['user_id']}&password={credentials['password']}"
        response = session.get(link, timeout=30)
        if response.status_code == 200:
            token = response.content.decode().strip()
            if token:
                log("Token received")
                return token
        return None
    except Exception as e:
        log(f"ERROR get_token: {e}")
        return None


def get_subscription_countries(token):
    """Получение списка стран"""
    try:
        response = session.get(f"{BASE_URL}/countries?token={token}", timeout=30)
        if response.status_code == 200:
            data = response.json()
            countries = []
            if isinstance(data, list):
                for item in data:
                    if isinstance(item, dict):
                        country_data = item.get('country', item)
                        if country_data.get('reportingcountry') == 'true' and country_data.get('defaultReporter') == True:
                            code = country_data.get('reportercode', '')
                            if code:
                                countries.append(code)
            countries = list(dict.fromkeys(countries))
            log(f"Available reporters: {len(countries)}")
            return countries
        return []
    except Exception as e:
        log(f"ERROR get_countries: {e}")
        return []


def parse_period(year, month):
    """Период → YYYY-MM-DD"""
    try:
        year = int(year)
        month = int(month)
        last_day = monthrange(year, month)[1]
        return datetime(year, month, last_day).strftime('%Y-%m-%d')
    except:
        return '2000-01-01'


def parse_record_to_dict(record, reporter_code="", trade_flow=""):
    """Парсинг записи в словарь"""
    try:
        if not isinstance(record, dict):
            return None
        year = record.get('year', 2000)
        month = record.get('month', 1)
        return {
            'ID': generate_id(),
            'Reporter_Code': record.get('reporterCode', reporter_code),
            'Trade_Flow': record.get('tradeFlow', trade_flow),
            'Is_Mirror_Data': record.get('isMirror', ''),
            'Period': parse_period(year, month),
            'Reporter_Name': record.get('reporterName', ''),
            'Reporter_Description': record.get('reporterDescription', '')[:120],
            'Reporter_Source': record.get('reporterSource', '')[:120],
            'Incoterm': record.get('incoterm', '')[:60],
            'Partner_Code': record.get('partnerCode', '')[:60],
            'Partner_Name': record.get('partnerName', '')[:120],
            'Commodity_HS_Code': record.get('hsCode', '')[:60],
            'Commodity_Description': record.get('commodityDescriptionOriginal', '')[:250],
            'HS6_Code': record.get('hs6Code', '')[:60],
            'HS6_Code_Description': record.get('hs6DescriptionOriginal', '')[:250],
            'Subdivision': record.get('subdivision', '')[:120],
            'Port': record.get('port', '')[:120],
            'Transport': record.get('transport', '')[:120],
            'Foreign_Port': record.get('foreignPort', '')[:120],
            'US_State': record.get('usState', '')[:120],
            'Customs_Regime': record.get('customsRegime', '')[:120],
            'Suppression': record.get('suppression', '')[:3900],
            'Monetary_Value': record.get('value', 0),
            'Currency': record.get('currency', 'USD')[:60],
            'Secondary_Monetary_Value': record.get('value2', 0),
            'Secondary_Currency': record.get('currency2', '')[:60],
            'Primary_Quantity': record.get('quantity1', 0),
            'Primary_Quantity_Unit': record.get('unit1', '')[:60],
            'Secondary_Quantity': record.get('quantity2', 0),
            'Secondary_Quantity_Unit': record.get('unit2', '')[:60],
            'Primary_Quantity_Price': record.get('price1', 0),
            'Primary_Quantity_Price_Unit': record.get('priceunit1', '')[:60],
            'Secondary_Quantity_Price': record.get('price2', 0),
            'Secondary_Quantity_Price_Unit': record.get('priceunit2', '')[:60],
        }
    except Exception as e:
        log(f"ERROR parse: {e}")
        return None


def parse_data_to_rows(data, reporter_code="", trade_flow=""):
    """Парсит data в список словарей"""
    rows = []

    if isinstance(data, list):
        records = data
    elif isinstance(data, dict):
        if 'reportData' in data:
            records = data['reportData']
        elif 'data' in data:
            records = data['data']
        else:
            return rows
    else:
        return rows

    for record in records:
        if isinstance(record, dict):
            parsed = parse_record_to_dict(record, reporter_code, trade_flow)
            if parsed:
                rows.append(parsed)

    return rows


def save_csv(rows, filepath):
    """Сохранить строки в CSV (для проверки)"""
    if not rows:
        return
    with open(filepath, 'w', newline='', encoding='utf-8') as f:
        writer = csv.DictWriter(f, fieldnames=rows[0].keys())
        writer.writeheader()
        writer.writerows(rows)
    log(f"CSV saved: {filepath} ({len(rows)} rows)")


def generate_xml(rows):
    """Генерирует XML из rows (как etl_common.py)"""
    xml_rows = []

    for row in rows:
        # Формируем <row><field>value</field>...</row>
        row_elems = []
        for key, value in row.items():
            # Заменяем пробелы и спецсимволы в тегах
            tag = str(key).replace(' ', '_')
            # Экранирование
            if value is None:
                value = ''
            row_elems.append(E(tag, str(value)))

        xml_rows.append(E.row(*row_elems))

    # Корневой элемент
    root = E.madabContent(
        E.type('GTT'),
        *xml_rows
    )

    return etree.tostring(root, encoding='utf-8', pretty_print=True)


def get_historical_data(token, countries, target_rows=100000):
    """Парсит target_rows строк"""

    log("=" * 60)
    log(f"Starting test parsing: {target_rows} rows")
    log("=" * 60)

    all_rows = []
    total_requests = 0
    hs_filter = '31,25'

    fields = 'reporterCode,tradeFlow,isMirror,year,month,reporterName,reporterDescription,reporterSource,incoterm,partnerCode,partnerName,hsCode,commodityDescriptionOriginal,hs6Code,hs6DescriptionOriginal,value,currency,value2,currency2,quantity1,unit1,quantity2,unit2,price1,priceunit1,price2,priceunit2,port,transport,subdivision'

    trade_details = 'PORT,SUBDIVISION,TRANSPORT,CUSTOMS_REGIME,FOREIGN_PORT,FOREIGN_SUBDIVISION'

    for country in countries:
        if len(all_rows) >= target_rows:
            log(f"Reached target rows: {len(all_rows)}")
            break

        for impexp in ['E', 'I']:
            if len(all_rows) >= target_rows:
                break

            try:
                params = {
                    'token': token,
                    'hscode': hs_filter,
                    'reporter': country,
                    'impexp': impexp,
                    'from': '2013-01',
                    'to': '2026-08',
                    'layout': 'brief',
                    'hslevel': '6',
                    'field': fields,
                    'tradedetails': trade_details,
                    'currency2': 'EUR',
                    'decimalscale': '2',
                    'format': 'json'
                }

                total_requests += 1

                start = time.time()
                response = session.get(f"{BASE_URL}/getreport", params=params, timeout=600)
                end = time.time()

                log(f"[{total_requests}] {country} {impexp} Status={response.status_code} Time={end-start:.1f}s Total rows={len(all_rows)}")

                if response.status_code == 200:
                    data = response.json()
                    new_rows = parse_data_to_rows(data, country, impexp)

                    # Добавляем только до target_rows
                    remaining = target_rows - len(all_rows)
                    if len(new_rows) > remaining:
                        new_rows = new_rows[:remaining]

                    all_rows.extend(new_rows)
                    log(f"  Added {len(new_rows)} rows, total: {len(all_rows)}")

            except Exception as e:
                log(f"✗ {str(e)}")
                continue

    return all_rows


def main():
    """Основная функция"""

    log('--------------------- TEST SCRIPT started ------------------------------')

    try:
        # 1. Токен
        token = get_token()
        if not token:
            log("ERROR: Failed to get token")
            sys.exit(1)

        # 2. Страны
        countries = get_subscription_countries(token)
        if not countries:
            log("ERROR: Failed to get countries")
            sys.exit(1)

        # 3. Парсим 100k строк
        log(f"Parsing {TARGET_ROWS} rows...")
        rows = get_historical_data(token, countries, TARGET_ROWS)

        if not rows:
            log("ERROR: No rows parsed")
            sys.exit(1)

        log(f"Parsed {len(rows)} rows")

        # 4. Сохраняем CSV (для проверки)
        save_csv(rows, CSV_TEMP)

        # 5. Генерируем XML (как etl_common.py)
        log("Generating XML...")
        xml_bytes = generate_xml(rows)

        # 6. Сохраняем XML
        with open(OUTPUT_XML, 'wb') as f:
            f.write(xml_bytes)

        size_mb = len(xml_bytes) / 1024 / 1024
        log(f"XML saved: {OUTPUT_XML}")
        log(f"XML size: {size_mb:.2f} MB")
        log(f"Rows in XML: {len(rows)}")

        log('--------------------- TEST SCRIPT finished -----------------------------')
        log(f"Now you can try: curl -X POST 'https://e-router.eurochem.ru/procedure/WHDEV/umdap.etl.content/call?application=externalGTT' --data-binary @{OUTPUT_XML} -k -v")

    except Exception as e:
        log(f'Error: {traceback.format_exception(*sys.exc_info())}')
        sys.exit(1)


if __name__ == "__main__":
    main()