---

import builtins
import json
import os
import sys
import logging
import traceback
import requests
from datetime import datetime, timedelta
from calendar import monthrange
from argparse import ArgumentParser
from io import StringIO
import csv
import time
import gc
import ctypes
from multiprocessing import Process, Queue

from umdap_config import LOG_FOLDER, ENV_FOLDER, EROUTER_HOST, PRIMARY_MADAB, UMDAP_ETL_URL_LIST

APPLICATION_NAME = 'externalGTT'
LOG_FILE = os.path.join(LOG_FOLDER, "externalGTT.log")
CONFIG_FILE = os.path.join(ENV_FOLDER, "externalGtt.json")
CREDENTIALS_FILE = os.path.join(ENV_FOLDER, "config.json")

BASE_URL = 'https://www.globaltradetracker.com/api/rest'
builtins.BASE_URL = BASE_URL
builtins.XML_DATE_FMT = '%d.%m.%Y %H:%M:%S'
builtins.DELIMITER = ','

session = requests.Session()
session.headers.update({
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'application/json, text/plain, */*',
    'Accept-Language': 'en-US,en;q=0.9',
    'Accept-Encoding': 'gzip, deflate, br',
    'Connection': 'keep-alive',
})

last_update_file = os.path.join(ENV_FOLDER, "last_update.json")

BATCH_SIZE = 100000
batch_file = os.path.join(ENV_FOLDER, "current_batch.csv")
total_sent = 0
current_batch_count = 0

ALL_KEYS = [
    'Reporter_Code', 'Trade_Flow', 'Is_Mirror_Data', 'Period',
    'Reporter_Name', 'Reporter_Description', 'Reporter_Source',
    'Incoterm', 'Partner_Code', 'Partner_Name',
    'Commodity_HS_Code', 'Commodity_Description',
    'HS6_Code', 'HS6_Code_Description',
    'Subdivision', 'Port', 'Transport', 'Foreign_Port', 'US_State',
    'Customs_Regime', 'Suppression',
    'Monetary_Value', 'Currency',
    'Secondary_Monetary_Value', 'Secondary_Currency',
    'Primary_Quantity', 'Primary_Quantity_Unit',
    'Secondary_Quantity', 'Secondary_Quantity_Unit',
    'Primary_Quantity_Price', 'Primary_Quantity_Price_Unit',
    'Secondary_Quantity_Price', 'Secondary_Quantity_Price_Unit'
]


def log(msg=''):
    """Логирование с временной меткой"""
    str_with_time = f'{datetime.now().strftime("%Y/%m/%d %H:%M:%S")} {msg}'
    print(str_with_time)
    with open(LOG_FILE, "a", encoding="utf-8") as file:
        file.write(f'{str_with_time}\n')

builtins.log = log


def get_token():
    """Получение токена для API"""
    try:
        if not os.path.exists(CREDENTIALS_FILE):
            log(f"ERROR: credentials file not found: {CREDENTIALS_FILE}")
            return None

        with open(CREDENTIALS_FILE, 'r') as f:
            credentials = json.load(f)

        link = f"{BASE_URL}/gettoken?userid={credentials['user_id']}&password={credentials['password']}"
        response = session.get(link, timeout=30)

        if response.status_code == 200:
            token = response.content.decode().strip()
            if token:
                log("Token received successfully")
                return token
        return None
    except Exception as e:
        log(f"ERROR in get_token: {e}")
        return None


def get_subscription_countries(token):
    """Получение списка стран"""
    try:
        response = session.get(f"{BASE_URL}/countries?token={token}", timeout=30)
        if response.status_code == 200:
            data = response.json()
            countries = []
            if isinstance(data, list):
                for item in data:
                    if isinstance(item, dict):
                        country_data = item.get('country', item)
                        if country_data.get('reportingcountry') == 'true' and country_data.get('defaultReporter') == True:
                            code = country_data.get('reportercode', '')
                            if code:
                                countries.append(code)
            countries = list(dict.fromkeys(countries))
            log(f"Available default reporters ({len(countries)})")
            return countries
        return []
    except Exception as e:
        log(f"ERROR in get_subscription_countries: {e}")
        return []


def get_last_update_time():
    """Чтение времени последнего обновления"""
    try:
        if os.path.exists(last_update_file):
            with open(last_update_file, 'r') as f:
                data = json.load(f)
                return data.get('last_update', None)
        return None
    except Exception as e:
        log(f"ERROR in get_last_update_time: {e}")
        return None


def save_last_update_time(update_time):
    """Сохранение времени последнего обновления"""
    try:
        with open(last_update_file, 'w') as f:
            json.dump({'last_update': update_time}, f)
    except Exception as e:
        log(f"ERROR in save_last_update_time: {e}")


def parse_period(year, month):
    """Парсинг периода"""
    try:
        year = int(year)
        month = int(month)
        last_day = monthrange(year, month)[1]
        return datetime(year, month, last_day).strftime('%Y-%m-%d')
    except:
        return '2000-01-01'


def parse_record_to_dict(record, reporter_code="", trade_flow=""):
    """Парсинг записи из нового формата API"""
    try:
        if not isinstance(record, dict):
            return None

        year = record.get('year', 2000)
        month = record.get('month', 1)

        return {
            'Reporter_Code': record.get('reporterCode', reporter_code),
            'Trade_Flow': record.get('tradeFlow', trade_flow),
            'Is_Mirror_Data': record.get('isMirror', ''),
            'Period': parse_period(year, month),
            'Reporter_Name': record.get('reporterName', ''),
            'Reporter_Description': record.get('reporterDescription', '')[:120],
            'Reporter_Source': record.get('reporterSource', '')[:120],
            'Incoterm': record.get('incoterm', '')[:60],
            'Partner_Code': record.get('partnerCode', '')[:60],
            'Partner_Name': record.get('partnerName', '')[:120],
            'Commodity_HS_Code': record.get('hsCode', '')[:60],
            'Commodity_Description': record.get('commodityDescriptionOriginal', '')[:250],
            'HS6_Code': record.get('hs6Code', '')[:60],
            'HS6_Code_Description': record.get('hs6DescriptionOriginal', '')[:250],
            'Subdivision': record.get('subdivision', '')[:120],
            'Port': record.get('port', '')[:120],
            'Transport': record.get('transport', '')[:120],
            'Foreign_Port': record.get('foreignPort', '')[:120],
            'US_State': record.get('usState', '')[:120],
            'Customs_Regime': record.get('customsRegime', '')[:120],
            'Suppression': record.get('suppression', '')[:3900],
            'Monetary_Value': record.get('value', 0),
            'Currency': record.get('currency', 'USD')[:60],
            'Secondary_Monetary_Value': record.get('value2', 0),
            'Secondary_Currency': record.get('currency2', '')[:60],
            'Primary_Quantity': record.get('quantity1', 0),
            'Primary_Quantity_Unit': record.get('unit1', '')[:60],
            'Secondary_Quantity': record.get('quantity2', 0),
            'Secondary_Quantity_Unit': record.get('unit2', '')[:60],
            'Primary_Quantity_Price': record.get('price1', 0),
            'Primary_Quantity_Price_Unit': record.get('priceunit1', '')[:60],
            'Secondary_Quantity_Price': record.get('price2', 0),
            'Secondary_Quantity_Price_Unit': record.get('priceunit2', '')[:60],
        }
    except:
        return None


def create_batch_file():
    """Создаёт новый CSV файл с заголовком"""
    if os.path.exists(batch_file):
        os.remove(batch_file)

    with open(batch_file, 'w', newline='', encoding='utf-8') as f:
        writer = csv.DictWriter(f, fieldnames=ALL_KEYS)
        writer.writeheader()


def write_to_batch(record):
    """Записывает одну запись в файл"""
    global current_batch_count

    with open(batch_file, 'a', newline='', encoding='utf-8') as f:
        writer = csv.DictWriter(f, fieldnames=ALL_KEYS)
        writer.writerow(record)

    current_batch_count += 1


def send_file_in_subprocess(file_path, queue):
    """Отправка файла в отдельном процессе"""
    try:
        from etl_common import process_etls

        def batch_request(url):
            with open(file_path, 'r') as f:
                return f.read()

        process_etls(
            application_name=APPLICATION_NAME,
            etl_desc_url=f'{EROUTER_HOST}/procedure/{PRIMARY_MADAB}/umdap.etl.description/call.json?application={APPLICATION_NAME}',
            data_type=None,
            api_requestor=batch_request,
            erouter_url_list=UMDAP_ETL_URL_LIST
        )

        queue.put(('success', 'ok'))
    except Exception as e:
        queue.put(('error', str(e)))


def send_batch_file():
    """Отправляет текущий файл в DWH"""
    global total_sent, current_batch_count

    if current_batch_count == 0:
        return

    log(f"Sending batch of {current_batch_count} records...")

    queue = Queue()
    p = Process(target=send_file_in_subprocess, args=(batch_file, queue))
    p.start()
    p.join(timeout=600)

    if p.is_alive():
        log("✗ Timeout! Terminating...")
        p.terminate()
        p.join()
    else:
        if not queue.empty():
            status, result = queue.get()
            if status == 'success':
                total_sent += current_batch_count
                log(f"✓ Batch sent. Total sent: {total_sent}")
            else:
                log(f"✗ Error: {result}")

    current_batch_count = 0
    create_batch_file()
    gc.collect()

    try:
        libc = ctypes.CDLL("libc.so.6")
        libc.malloc_trim(0)
        log("Memory released to OS")
    except:
        pass


def add_record(record, reporter_code="", trade_flow=""):
    """Обрабатывает одну запись"""
    parsed = parse_record_to_dict(record, reporter_code, trade_flow)
    if parsed:
        write_to_batch(parsed)

        if current_batch_count >= BATCH_SIZE:
            send_batch_file()


def process_records(records, reporter_code="", trade_flow=""):
    """Обрабатывает список записей"""
    for record in records:
        if isinstance(record, dict):
            add_record(record, reporter_code, trade_flow)


def get_historical_data(token, countries):
    """БЛОК 1: Исторический сбор с 2013 по 2026"""
    log("=" * 60)
    log("BLOCK 1: Starting historical data collection from 2013")
    log(f"Countries ({len(countries)}), HS Codes: 31,25")
    log("=" * 60)

    create_batch_file()

    total_requests = 0
    hs_filter = '31,25'

    fields = 'reporterCode,tradeFlow,isMirror,year,month,reporterName,reporterDescription,reporterSource,incoterm,partnerCode,partnerName,hsCode,commodityDescriptionOriginal,hs6Code,hs6DescriptionOriginal,value,currency,value2,currency2,quantity1,unit1,quantity2,unit2,price1,priceunit1,price2,priceunit2,port,transport,subdivision'

    trade_details = 'PORT,SUBDIVISION,TRANSPORT,CUSTOMS_REGIME,FOREIGN_PORT,FOREIGN_SUBDIVISION'

    for country in countries:
        for impexp in ['E', 'I']:
            try:
                params = {
                    'token': token,
                    'hscode': hs_filter,
                    'reporter': country,
                    'impexp': impexp,
                    'from': '2013-01',
                    'to': '2026-08',
                    'layout': 'brief',
                    'hslevel': '6',
                    'field': fields,
                    'tradedetails': trade_details,
                    'currency2': 'EUR',
                    'decimalscale': '2',
                    'format': 'json'
                }

                total_requests += 1

                start_time = time.time()
                response = session.get(f"{BASE_URL}/getreport", params=params, timeout=600)
                end_time = time.time()

                log(f"[{total_requests}] {country} HS={hs_filter} {impexp} Status={response.status_code} Time={end_time-start_time:.1f}s Batch={current_batch_count} Sent={total_sent}")

                if response.status_code == 200:
                    data = response.json()

                    if isinstance(data, list) and len(data) > 0:
                        process_records(data, country, impexp)
                    elif isinstance(data, dict):
                        if 'reportData' in data:
                            process_records(data['reportData'], country, impexp)
                        elif 'data' in data:
                            process_records(data['data'], country, impexp)

            except Exception as e:
                log(f"✗ {str(e)}")
                continue

    if current_batch_count > 0:
        send_batch_file()

    if os.path.exists(batch_file):
        os.remove(batch_file)

    log(f"\nCOMPLETED: {total_requests} requests, {total_sent} records sent")


def get_updated_data(token, countries):
    """БЛОК 2: Регулярное обновление"""
    log("=" * 60)
    log("BLOCK 2: Checking for updates")
    log("=" * 60)

    last_update = get_last_update_time()
    current_date = datetime.now().strftime('%Y-%m')

    create_batch_file()

    hs_filter = '31,25'

    fields = 'reporterCode,tradeFlow,isMirror,year,month,reporterName,reporterDescription,reporterSource,incoterm,partnerCode,partnerName,hsCode,commodityDescriptionOriginal,hs6Code,hs6DescriptionOriginal,value,currency,value2,currency2,quantity1,unit1,quantity2,unit2,price1,priceunit1,price2,priceunit2,port,transport,subdivision'

    trade_details = 'PORT,SUBDIVISION,TRANSPORT,CUSTOMS_REGIME,FOREIGN_PORT,FOREIGN_SUBDIVISION'

    for country in countries:
        for impexp in ['E', 'I']:
            try:
                params = {
                    'token': token,
                    'hscode': hs_filter,
                    'reporter': country,
                    'impexp': impexp,
                    'updatedAfter': last_update if last_update else '2013-01',
                    'latestavailablemonths': 3,
                    'layout': 'brief',
                    'hslevel': '6',
                    'field': fields,
                    'tradedetails': trade_details,
                    'currency2': 'EUR',
                    'decimalscale': '2',
                    'format': 'json'
                }

                response = session.get(f"{BASE_URL}/getreport", params=params, timeout=300)
                if response.status_code == 200:
                    data = response.json()
                    if isinstance(data, list) and len(data) > 0:
                        process_records(data, country, impexp)
                    elif isinstance(data, dict) and 'reportData' in data:
                        process_records(data['reportData'], country, impexp)

            except Exception as e:
                log(f"✗ {str(e)}")
                continue

    if current_batch_count > 0:
        send_batch_file()

    if os.path.exists(batch_file):
        os.remove(batch_file)

    save_last_update_time(current_date)

    log(f"Updated. Total sent: {total_sent}")


def main():
    """Основная функция"""
    log('--------------------- Script started ------------------------------')

    try:
        token = get_token()
        if not token:
            log("ERROR: Failed to get token")
            sys.exit(1)

        countries = get_subscription_countries(token)

        if not countries:
            log("ERROR: Failed to get parameters")
            sys.exit(1)

        last_update = get_last_update_time()

        if last_update is None:
            log("First run - BLOCK 1: Historical collection")
            get_historical_data(token, countries)
            save_last_update_time(datetime.now().strftime('%Y-%m'))
        else:
            log(f"Regular run - BLOCK 2: Update (last: {last_update})")
            get_updated_data(token, countries)

        log(f"Total sent: {total_sent}")

    except Exception as e:
        log(f'Error: {traceback.format_exception(*sys.exc_info())}')
        sys.exit(1)
    finally:
        log('--------------------- Script finished -----------------------------')


if __name__ == "__main__":
    main()