---

import builtins
import json
import os
import sys
import logging
import traceback
import requests
import uuid
from datetime import datetime, timedelta
from calendar import monthrange
from argparse import ArgumentParser
from io import StringIO
import csv
import time
import gc
import ctypes
from lxml.builder import E
from lxml import etree

from umdap_config import LOG_FOLDER, ENV_FOLDER, EROUTER_HOST, PRIMARY_MADAB, UMDAP_ETL_URL_LIST

APPLICATION_NAME = 'externalGTT'
LOG_FILE = os.path.join(LOG_FOLDER, "externalGTT.log")
CONFIG_FILE = os.path.join(ENV_FOLDER, "externalGtt.json")
CREDENTIALS_FILE = os.path.join(ENV_FOLDER, "config.json")

BASE_URL = 'https://www.globaltradetracker.com/api/rest'
builtins.BASE_URL = BASE_URL
builtins.XML_DATE_FMT = '%d.%m.%Y %H:%M:%S'
builtins.DELIMITER = ','

session = requests.Session()
session.headers.update({
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'application/json, text/plain, */*',
    'Accept-Language': 'en-US,en;q=0.9',
    'Accept-Encoding': 'gzip, deflate, br',
    'Connection': 'keep-alive',
})

last_update_file = os.path.join(ENV_FOLDER, "last_update.json")

BATCH_SIZE = 100000
WRITE_BUFFER_SIZE = 5000
batch_file = os.path.join(ENV_FOLDER, "current_batch.csv")
total_sent = 0
current_batch_count = 0

write_buffer = []

# === ДЛЯ ТЕСТА: сколько строк спарсить ===
TARGET_ROWS = 100000
OUTPUT_XML = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'test.xml')

# Все страны, которые УЖЕ спарсились (по логам)
SKIP_COUNTRIES = [
    # Первый запуск (до CA_national)
    'AD_comtrade', 'AE_comtrade', 'AF_comtrade', 'AG_comtrade', 'AI_comtrade',
    'AL_eurostat', 'AM_comtrade', 'AN_comtrade', 'AO_national', 'AR_national',
    'AT_national', 'AU_national', 'AW_comtrade', 'AZ_comtrade', 'BA_eurostat',
    'BB_comtrade', 'BD_national', 'BE_eurostat', 'BF_comtrade', 'BG_eurostat',
    'BH_national', 'BI_comtrade', 'BJ_comtrade', 'BM_comtrade', 'BN_comtrade',
    'BO_national', 'BR_national', 'BS_comtrade', 'BT_comtrade', 'BW_national',
    'BX_eurostat', 'BY_national', 'BZ_national', 'CA_national',
    # Второй запуск (после CA_national до ES_national)
    'CD_comtrade', 'CF_comtrade', 'CG_comtrade', 'CH_national',
    'CI_national', 'CK_comtrade', 'CL_national', 'CM_national',
    'CN_national', 'CO_national', 'CR_national', 'CU_comtrade',
    'CV_comtrade', 'CY_national', 'CZ_national', 'DE_national',
    'DJ_comtrade', 'DK_national', 'DM_comtrade', 'DO_national',
    'DZ_national', 'EC_national', 'EE_national', 'EG_national',
    'ER_comtrade', 'ES_national',
    # Третий запуск (до GB_national E)
    'ET_national', 'FI_national', 'FJ_comtrade', 'FM_comtrade',
    'FO_national', 'FR_national', 'GA_comtrade', 'GB_national',
    # Четвёртый запуск (после GB_national E до IR_national E)
    'GD_comtrade', 'GE_national', 'GH_national', 'GL_comtrade',
    'GM_comtrade', 'GN_comtrade', 'GR_eurostat', 'GT_national',
    'GW_comtrade', 'GY_comtrade', 'HK_national', 'HN_national',
    'HR_eurostat', 'HU_eurostat', 'ID_national', 'IE_national',
    'IL_national', 'IN_national', 'IQ_comtrade', 'IR_national',
    # Пятый запуск (до JP_national I)
    'IS_national', 'IT_national', 'JM_comtrade', 'JO_national', 'JP_national',
    # Шестой запуск (после JP_national I до LU_eurostat I)
    'KE_national', 'KG_comtrade', 'KH_comtrade', 'KI_comtrade',
    'KM_comtrade', 'KN_comtrade', 'KR_national', 'KW_comtrade',
    'KY_comtrade', 'KZ_national', 'LA_comtrade', 'LB_comtrade',
    'LC_comtrade', 'LK_national', 'LS_comtrade', 'LT_national',
    'LU_eurostat',
    # Седьмой запуск (после LU_eurostat I до NL_eurostat I)
    'LV_national', 'LY_comtrade', 'MA_national', 'MD_comtrade',
    'MG_comtrade', 'MK_eurostat', 'ML_comtrade', 'MM_comtrade',
    'MN_comtrade', 'MO_national', 'MR_comtrade', 'MS_comtrade',
    'MT_eurostat', 'MU_national', 'MV_comtrade', 'MW_national',
    'MX_national', 'MY_national',
    # Восьмой запуск (после MY_national I до NL_eurostat I)
    'MZ_national', 'NA_comtrade', 'NC_comtrade', 'NE_comtrade',
    'NG_national', 'NI_national', 'NL_eurostat',
    # Девятый запуск (до RU_national I)
    'NO_national', 'NP_national', 'NZ_national', 'OM_comtrade',
    'PA_national', 'PE_national', 'PF_comtrade', 'PG_comtrade',
    'PH_national', 'PK_national', 'PL_national', 'PR_national',
    'PS_comtrade', 'PT_national', 'PW_comtrade', 'PY_national',
    'QA_national', 'RO_eurostat', 'RU_national',
    # Десятый запуск (после RU_national I до US_district E)
    'RW_comtrade', 'SA_national', 'SB_comtrade', 'SC_comtrade',
    'SD_comtrade', 'SE_national', 'SG_national', 'SI_national',
    'SK_eurostat',
    # Одиннадцатый запуск (после SK_eurostat I до US_district E)
    'SL_comtrade', 'SN_national', 'SR_comtrade', 'SS_comtrade',
    'ST_comtrade', 'SV_national', 'SY_comtrade', 'SZ_national',
    'TC_comtrade', 'TG_comtrade', 'TH_national', 'TJ_comtrade',
    'TL_comtrade', 'TM_comtrade', 'TN_comtrade', 'TO_comtrade',
    'TR_national',
    # Двенадцатый запуск (после TR_national E до US_district E)
    'TT_comtrade', 'TV_comtrade', 'TW_national', 'TZ_comtrade',
    'UA_national', 'UG_comtrade', 'US_district'
]

ALL_KEYS = [
    'ID',
    'Reporter_Code', 'Trade_Flow', 'Is_Mirror_Data', 'Period',
    'Reporter_Name', 'Reporter_Description', 'Reporter_Source',
    'Incoterm', 'Partner_Code', 'Partner_Name',
    'Commodity_HS_Code', 'Commodity_Description',
    'HS6_Code', 'HS6_Code_Description',
    'Subdivision', 'Port', 'Transport', 'Foreign_Port', 'US_State',
    'Customs_Regime', 'Suppression',
    'Monetary_Value', 'Currency',
    'Secondary_Monetary_Value', 'Secondary_Currency',
    'Primary_Quantity', 'Primary_Quantity_Unit',
    'Secondary_Quantity', 'Secondary_Quantity_Unit',
    'Primary_Quantity_Price', 'Primary_Quantity_Price_Unit',
    'Secondary_Quantity_Price', 'Secondary_Quantity_Price_Unit'
]


def log(msg=''):
    """Логирование с временной меткой"""
    str_with_time = f'{datetime.now().strftime("%Y/%m/%d %H:%M:%S")} {msg}'
    print(str_with_time)
    with open(LOG_FILE, "a", encoding="utf-8") as file:
        file.write(f'{str_with_time}\n')

builtins.log = log


def generate_id():
    """Генерация 32-символьного ID в нижнем регистре"""
    return uuid.uuid4().hex


def get_token():
    """Получение токена для API"""
    try:
        if not os.path.exists(CREDENTIALS_FILE):
            log(f"ERROR: credentials file not found: {CREDENTIALS_FILE}")
            return None

        with open(CREDENTIALS_FILE, 'r') as f:
            credentials = json.load(f)

        link = f"{BASE_URL}/gettoken?userid={credentials['user_id']}&password={credentials['password']}"
        response = session.get(link, timeout=30)

        if response.status_code == 200:
            token = response.content.decode().strip()
            if token:
                log("Token received successfully")
                return token
        return None
    except Exception as e:
        log(f"ERROR in get_token: {e}")
        return None


def get_subscription_countries(token):
    """Получение списка стран"""
    try:
        response = session.get(f"{BASE_URL}/countries?token={token}", timeout=30)
        if response.status_code == 200:
            data = response.json()
            countries = []
            if isinstance(data, list):
                for item in data:
                    if isinstance(item, dict):
                        country_data = item.get('country', item)
                        if country_data.get('reportingcountry') == 'true' and country_data.get('defaultReporter') == True:
                            code = country_data.get('reportercode', '')
                            if code:
                                countries.append(code)
            countries = list(dict.fromkeys(countries))
            log(f"Available default reporters ({len(countries)})")
            return countries
        return []
    except Exception as e:
        log(f"ERROR in get_subscription_countries: {e}")
        return []


def get_last_update_time():
    """Чтение времени последнего обновления"""
    try:
        if os.path.exists(last_update_file):
            with open(last_update_file, 'r') as f:
                data = json.load(f)
                return data.get('last_update', None)
        return None
    except Exception as e:
        log(f"ERROR in get_last_update_time: {e}")
        return None


def save_last_update_time(update_time):
    """Сохранение времени последнего обновления"""
    try:
        with open(last_update_file, 'w') as f:
            json.dump({'last_update': update_time}, f)
    except Exception as e:
        log(f"ERROR in save_last_update_time: {e}")


def parse_period(year, month):
    """Парсинг периода"""
    try:
        year = int(year)
        month = int(month)
        last_day = monthrange(year, month)[1]
        return datetime(year, month, last_day).strftime('%Y-%m-%d')
    except:
        return '2000-01-01'


def parse_record_to_dict(record, reporter_code="", trade_flow=""):
    """Парсинг записи из нового формата API"""
    try:
        if not isinstance(record, dict):
            return None

        year = record.get('year', 2000)
        month = record.get('month', 1)

        return {
            'ID': generate_id(),
            'Reporter_Code': record.get('reporterCode', reporter_code),
            'Trade_Flow': record.get('tradeFlow', trade_flow),
            'Is_Mirror_Data': record.get('isMirror', ''),
            'Period': parse_period(year, month),
            'Reporter_Name': record.get('reporterName', ''),
            'Reporter_Description': record.get('reporterDescription', '')[:120],
            'Reporter_Source': record.get('reporterSource', '')[:120],
            'Incoterm': record.get('incoterm', '')[:60],
            'Partner_Code': record.get('partnerCode', '')[:60],
            'Partner_Name': record.get('partnerName', '')[:120],
            'Commodity_HS_Code': record.get('hsCode', '')[:60],
            'Commodity_Description': record.get('commodityDescriptionOriginal', '')[:250],
            'HS6_Code': record.get('hs6Code', '')[:60],
            'HS6_Code_Description': record.get('hs6DescriptionOriginal', '')[:250],
            'Subdivision': record.get('subdivision', '')[:120],
            'Port': record.get('port', '')[:120],
            'Transport': record.get('transport', '')[:120],
            'Foreign_Port': record.get('foreignPort', '')[:120],
            'US_State': record.get('usState', '')[:120],
            'Customs_Regime': record.get('customsRegime', '')[:120],
            'Suppression': record.get('suppression', '')[:3900],
            'Monetary_Value': record.get('value', 0),
            'Currency': record.get('currency', 'USD')[:60],
            'Secondary_Monetary_Value': record.get('value2', 0),
            'Secondary_Currency': record.get('currency2', '')[:60],
            'Primary_Quantity': record.get('quantity1', 0),
            'Primary_Quantity_Unit': record.get('unit1', '')[:60],
            'Secondary_Quantity': record.get('quantity2', 0),
            'Secondary_Quantity_Unit': record.get('unit2', '')[:60],
            'Primary_Quantity_Price': record.get('price1', 0),
            'Primary_Quantity_Price_Unit': record.get('priceunit1', '')[:60],
            'Secondary_Quantity_Price': record.get('price2', 0),
            'Secondary_Quantity_Price_Unit': record.get('priceunit2', '')[:60],
        }
    except Exception as e:
        log(f"ERROR in parse_record_to_dict: {e}")
        return None


def create_batch_file():
    """Создаёт новый CSV файл с заголовком"""
    if os.path.exists(batch_file):
        os.remove(batch_file)

    with open(batch_file, 'w', newline='', encoding='utf-8') as f:
        writer = csv.DictWriter(f, fieldnames=ALL_KEYS)
        writer.writeheader()


def flush_write_buffer():
    """Сбрасывает буфер на диск"""
    global write_buffer

    if not write_buffer:
        return

    with open(batch_file, 'a', newline='', encoding='utf-8') as f:
        writer = csv.DictWriter(f, fieldnames=ALL_KEYS)
        writer.writerows(write_buffer)

    write_buffer = []


def write_to_batch(record):
    """Записывает запись в буфер, при заполнении — на диск"""
    global current_batch_count

    write_buffer.append(record)
    current_batch_count += 1

    if len(write_buffer) >= WRITE_BUFFER_SIZE:
        flush_write_buffer()


def generate_xml_from_csv(csv_path):
    """Генерирует XML из CSV (как etl_common.py)"""
    rows = []
    with open(csv_path, 'r', encoding='utf-8') as f:
        reader = csv.DictReader(f)
        for row in reader:
            rows.append(row)

    log(f"Read {len(rows)} rows from CSV")

    xml_rows = []
    for row in rows:
        row_elems = []
        for key, value in row.items():
            tag = str(key).replace(' ', '_')
            if value is None:
                value = ''
            row_elems.append(E(tag, str(value)))
        xml_rows.append(E.row(*row_elems))

    root = E.madabContent(
        E.type('GTT'),
        *xml_rows
    )

    return etree.tostring(root, encoding='utf-8', pretty_print=True)


def save_xml_before_send():
    """Сохраняет XML вместо отправки"""
    global total_sent, current_batch_count

    flush_write_buffer()

    if current_batch_count == 0:
        return

    log(f"Saving batch of {current_batch_count} records to XML (NOT sending)...")

    # Генерируем XML из CSV
    try:
        xml_bytes = generate_xml_from_csv(batch_file)

        with open(OUTPUT_XML, 'wb') as f:
            f.write(xml_bytes)

        size_mb = len(xml_bytes) / 1024 / 1024
        log(f"✓ XML saved: {OUTPUT_XML}")
        log(f"  Size: {size_mb:.2f} MB")
        log(f"  Rows: {current_batch_count}")
        log(f"  NOT SENT to e-router!")

        total_sent += current_batch_count

    except Exception as e:
        log(f"✗ Error saving XML: {e}")

    current_batch_count = 0
    create_batch_file()
    gc.collect()

    try:
        libc = ctypes.CDLL("libc.so.6")
        libc.malloc_trim(0)
        log("Memory released to OS")
    except:
        pass


def add_record(record, reporter_code="", trade_flow=""):
    """Обрабатывает одну запись"""
    global current_batch_count

    parsed = parse_record_to_dict(record, reporter_code, trade_flow)
    if parsed:
        write_to_batch(parsed)

        # Достигли TARGET_ROWS — сохраняем XML и выходим
        if current_batch_count >= TARGET_ROWS:
            log(f"Reached TARGET_ROWS: {TARGET_ROWS}")
            save_xml_before_send()
            return True  # сигнал остановки

    return False


def process_records(records, reporter_code="", trade_flow=""):
    """Обрабатывает список записей"""
    for record in records:
        if isinstance(record, dict):
            if add_record(record, reporter_code, trade_flow):
                return True  # остановка
    return False


def get_historical_data(token, countries):
    """БЛОК 1: Исторический сбор с 2013 по 2026 (с остановкой на TARGET_ROWS)"""

    log("=" * 60)
    log("BLOCK 1: Starting historical data collection from 2013")
    log(f"Countries ({len(countries)}), HS Codes: 31,25")
    log(f"Skip countries: {len(SKIP_COUNTRIES)}")
    log(f"TARGET_ROWS: {TARGET_ROWS}")
    log(f"BATCH_SIZE: {BATCH_SIZE}")
    log("=" * 60)

    create_batch_file()

    total_requests = 0
    skipped_count = 0
    hs_filter = '31,25'

    fields = 'reporterCode,tradeFlow,isMirror,year,month,reporterName,reporterDescription,reporterSource,incoterm,partnerCode,partnerName,hsCode,commodityDescriptionOriginal,hs6Code,hs6DescriptionOriginal,value,currency,value2,currency2,quantity1,unit1,quantity2,unit2,price1,priceunit1,price2,priceunit2,port,transport,subdivision'

    trade_details = 'PORT,SUBDIVISION,TRANSPORT,CUSTOMS_REGIME,FOREIGN_PORT,FOREIGN_SUBDIVISION'

    for country in countries:

        if country in SKIP_COUNTRIES:
            skipped_count += 1
            continue

        for impexp in ['E', 'I']:
            try:
                params = {
                    'token': token,
                    'hscode': hs_filter,
                    'reporter': country,
                    'impexp': impexp,
                    'from': '2013-01',
                    'to': '2026-08',
                    'layout': 'brief',
                    'hslevel': '6',
                    'field': fields,
                    'tradedetails': trade_details,
                    'currency2': 'EUR',
                    'decimalscale': '2',
                    'format': 'json'
                }

                total_requests += 1

                start_time = time.time()
                response = session.get(f"{BASE_URL}/getreport", params=params, timeout=600)
                end_time = time.time()

                log(f"[{total_requests}] {country} HS={hs_filter} {impexp} Status={response.status_code} Time={end_time-start_time:.1f}s Batch={current_batch_count}")

                if response.status_code == 200:
                    data = response.json()

                    if isinstance(data, list) and len(data) > 0:
                        if process_records(data, country, impexp):
                            return  # остановка
                    elif isinstance(data, dict):
                        if 'reportData' in data:
                            if process_records(data['reportData'], country, impexp):
                                return
                        elif 'data' in data:
                            if process_records(data['data'], country, impexp):
                                return

            except Exception as e:
                log(f"✗ {str(e)}")
                continue

    # Если TARGET_ROWS не достигнут — сохраняем остаток
    if current_batch_count > 0:
        save_xml_before_send()

    if os.path.exists(batch_file):
        os.remove(batch_file)

    log(f"\nCOMPLETED: {total_requests} requests, {total_sent} records saved")


def main():
    """Основная функция"""
    log('--------------------- TEST SCRIPT started ------------------------------')

    try:
        token = get_token()
        if not token:
            log("ERROR: Failed to get token")
            sys.exit(1)

        countries = get_subscription_countries(token)

        if not countries:
            log("ERROR: Failed to get parameters")
            sys.exit(1)

        log(f"Parsing {TARGET_ROWS} rows (NOT sending to e-router)...")
        get_historical_data(token, countries)

        log(f"Total records saved to XML: {total_sent}")
        log(f"Output: {OUTPUT_XML}")
        log('--------------------- TEST SCRIPT finished -----------------------------')
        log(f"Try: curl -X POST 'https://e-router.eurochem.ru/procedure/WHDEV/umdap.etl.content/call?application=externalGTT' --data-binary @{OUTPUT_XML} -k -v")

    except Exception as e:
        log(f'Error: {traceback.format_exception(*sys.exc_info())}')
        sys.exit(1)


if __name__ == "__main__":
    main()