---

import builtins
import json
import os
import sys
import logging
import traceback
import requests
from datetime import datetime, timedelta
from calendar import monthrange
from argparse import ArgumentParser
from io import StringIO
import csv
import time
import gc
from multiprocessing import Process, Queue

from umdap_config import LOG_FOLDER, ENV_FOLDER, EROUTER_HOST, PRIMARY_MADAB, UMDAP_ETL_URL_LIST

APPLICATION_NAME = 'externalGTT'
LOG_FILE = os.path.join(LOG_FOLDER, "externalGTT.log")
CONFIG_FILE = os.path.join(ENV_FOLDER, "externalGtt.json")
CREDENTIALS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.json")

BASE_URL = 'https://www.globaltradetracker.com/api/rest'
builtins.BASE_URL = BASE_URL
builtins.XML_DATE_FMT = '%d.%m.%Y %H:%M:%S'
builtins.DELIMITER = ','

session = requests.Session()
session.headers.update({
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'application/json, text/plain, */*',
    'Accept-Language': 'en-US,en;q=0.9',
    'Accept-Encoding': 'gzip, deflate, br',
    'Connection': 'keep-alive',
})

last_update_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "last_update.json")

BATCH_SIZE = 500000
batch = list()
total_sent = 0

ALL_KEYS = [
    'Reporter_Code', 'Trade_Flow', 'Is_Mirror_Data', 'Period',
    'Reporter_Name', 'Reporter_Description', 'Reporter_Source',
    'Incoterm', 'Partner_Code', 'Partner_Name',
    'Commodity_HS_Code', 'Commodity_Description',
    'HS6_Code', 'HS6_Code_Description',
    'Subdivision', 'Port', 'Transport', 'Foreign_Port', 'US_State',
    'Customs_Regime', 'Suppression',
    'Monetary_Value', 'Currency',
    'Secondary_Monetary_Value', 'Secondary_Currency',
    'Primary_Quantity', 'Primary_Quantity_Unit',
    'Secondary_Quantity', 'Secondary_Quantity_Unit',
    'Primary_Quantity_Price', 'Primary_Quantity_Price_Unit',
    'Secondary_Quantity_Price', 'Secondary_Quantity_Price_Unit'
]


def log(msg=''):
    """Логирование с временной меткой"""
    str_with_time = f'{datetime.now().strftime("%Y/%m/%d %H:%M:%S")} {msg}'
    print(str_with_time)
    with open(LOG_FILE, "a", encoding="utf-8") as file:
        file.write(f'{str_with_time}\n')

builtins.log = log


def get_token():
    """Получение токена для API"""
    try:
        with open(CREDENTIALS_FILE, 'r') as f:
            credentials = json.load(f)

        link = f"{BASE_URL}/gettoken?userid={credentials['user_id']}&password={credentials['password']}"
        response = session.get(link, timeout=30)

        if response.status_code == 200:
            token = response.content.decode().strip()
            if token:
                log("Token received successfully")
                return token
        return None
    except:
        return None


def get_subscription_countries(token):
    """Получение списка стран"""
    try:
        response = session.get(f"{BASE_URL}/countries?token={token}", timeout=30)
        if response.status_code == 200:
            data = response.json()
            countries = []
            if isinstance(data, list):
                for item in data:
                    if isinstance(item, dict):
                        country_data = item.get('country', item)
                        if country_data.get('reportingcountry') == 'true' and country_data.get('defaultReporter') == True:
                            code = country_data.get('reportercode', '')
                            if code:
                                countries.append(code)
            countries = list(dict.fromkeys(countries))
            log(f"Available default reporters ({len(countries)})")
            return countries
        return []
    except:
        return []


def get_last_update_time():
    """Чтение времени последнего обновления"""
    try:
        if os.path.exists(last_update_file):
            with open(last_update_file, 'r') as f:
                data = json.load(f)
                return data.get('last_update', None)
        return None
    except:
        return None


def save_last_update_time(update_time):
    """Сохранение времени последнего обновления"""
    try:
        with open(last_update_file, 'w') as f:
            json.dump({'last_update': update_time}, f)
    except:
        pass


def parse_period(year, month):
    """Парсинг периода"""
    try:
        year = int(year)
        month = int(month)
        last_day = monthrange(year, month)[1]
        return datetime(year, month, last_day).strftime('%Y-%m-%d')
    except:
        return '2000-01-01'


def parse_record_to_dict(record, reporter_code="", trade_flow=""):
    """Парсинг записи из нового формата API"""
    try:
        if not isinstance(record, dict):
            return None

        year = record.get('year', 2000)
        month = record.get('month', 1)

        return {
            'Reporter_Code': record.get('reporterCode', reporter_code),
            'Trade_Flow': record.get('tradeFlow', trade_flow),
            'Is_Mirror_Data': record.get('isMirror', ''),
            'Period': parse_period(year, month),
            'Reporter_Name': record.get('reporterName', ''),
            'Reporter_Description': record.get('reporterDescription', ''),
            'Reporter_Source': record.get('reporterSource', ''),
            'Incoterm': record.get('incoterm', ''),
            'Partner_Code': record.get('partnerCode', ''),
            'Partner_Name': record.get('partnerName', ''),
            'Commodity_HS_Code': record.get('hsCode', ''),
            'Commodity_Description': record.get('commodityDescriptionOriginal', ''),
            'HS6_Code': record.get('hs6Code', ''),
            'HS6_Code_Description': record.get('hs6DescriptionOriginal', ''),
            'Subdivision': record.get('subdivision', ''),
            'Port': record.get('port', ''),
            'Transport': record.get('transport', ''),
            'Foreign_Port': record.get('foreignPort', ''),
            'US_State': record.get('usState', ''),
            'Customs_Regime': record.get('customsRegime', ''),
            'Suppression': record.get('suppression', ''),
            'Monetary_Value': record.get('value', 0),
            'Currency': record.get('currency', 'USD'),
            'Secondary_Monetary_Value': record.get('value2', 0),
            'Secondary_Currency': record.get('currency2', ''),
            'Primary_Quantity': record.get('quantity1', 0),
            'Primary_Quantity_Unit': record.get('unit1', ''),
            'Secondary_Quantity': record.get('quantity2', 0),
            'Secondary_Quantity_Unit': record.get('unit2', ''),
            'Primary_Quantity_Price': record.get('price1', 0),
            'Primary_Quantity_Price_Unit': record.get('priceunit1', ''),
            'Secondary_Quantity_Price': record.get('price2', 0),
            'Secondary_Quantity_Price_Unit': record.get('priceunit2', ''),
        }
    except:
        return None


def send_batch_in_subprocess(batch_data, queue):
    """Отправка батча в отдельном процессе"""
    temp_file = f'temp_batch_{os.getpid()}.csv'

    try:
        with open(temp_file, 'w', newline='', encoding='utf-8') as f:
            writer = csv.DictWriter(f, fieldnames=ALL_KEYS)
            writer.writeheader()
            writer.writerows(batch_data)

        from etl_common import process_etls

        def batch_request(url):
            with open(temp_file, 'r') as f:
                return f.read()

        process_etls(
            application_name=APPLICATION_NAME,
            etl_desc_url=f'{EROUTER_HOST}/procedure/{PRIMARY_MADAB}/umdap.etl.description/call.json?application={APPLICATION_NAME}',
            data_type=None,
            api_requestor=batch_request,
            erouter_url_list=UMDAP_ETL_URL_LIST
        )

        queue.put(('success', len(batch_data)))
    except Exception as e:
        queue.put(('error', str(e)))
    finally:
        if os.path.exists(temp_file):
            os.remove(temp_file)


def send_batch_to_dwh(batch_data):
    """Отправка батча в DWH через отдельный процесс"""
    global total_sent, batch

    if not batch_data:
        return

    log(f"Sending batch of {len(batch_data)} records...")

    queue = Queue()
    p = Process(target=send_batch_in_subprocess, args=(batch_data, queue))
    p.start()

    p.join(timeout=600)

    if p.is_alive():
        log("✗ Timeout! Terminating...")
        p.terminate()
        p.join()
    else:
        if not queue.empty():
            status, result = queue.get()
            if status == 'success':
                total_sent += result
                log(f"✓ Batch sent. Total sent: {total_sent}")
            else:
                log(f"✗ Error: {result}")

    batch_data = None
    batch = []
    gc.collect()


def add_to_batch(records, reporter_code="", trade_flow=""):
    """Добавление записей в порцию"""
    global batch

    for record in records:
        if isinstance(record, dict):
            parsed = parse_record_to_dict(record, reporter_code, trade_flow)
            if parsed:
                batch.append(parsed)
                if len(batch) >= BATCH_SIZE:
                    send_batch_to_dwh(batch)


def flush_batch():
    """Отправка оставшихся"""
    if batch:
        send_batch_to_dwh(batch)


def get_historical_data(token, countries):
    """БЛОК 1: Исторический сбор с 2020 по 2026"""
    log("=" * 60)
    log("BLOCK 1: Starting historical data collection from 2020")
    log(f"Countries ({len(countries)})")
    log("=" * 60)

    total_requests = 0

    fields = 'reporterCode,tradeFlow,isMirror,year,month,reporterName,reporterDescription,reporterSource,incoterm,partnerCode,partnerName,hsCode,commodityDescriptionOriginal,hs6Code,hs6DescriptionOriginal,value,currency,value2,currency2,quantity1,unit1,quantity2,unit2,price1,priceunit1,price2,priceunit2,port,transport,subdivision'

    trade_details = 'PORT,SUBDIVISION,TRANSPORT,CUSTOMS_REGIME,FOREIGN_PORT,FOREIGN_SUBDIVISION'

    for country in countries:
        for impexp in ['E', 'I']:
            try:
                params = {
                    'token': token,
                    'hscode': '31',
                    'reporter': country,
                    'impexp': impexp,
                    'from': '2020-01',
                    'to': '2026-08',
                    'layout': 'brief',
                    'hslevel': '6',
                    'field': fields,
                    'tradedetails': trade_details,
                    'currency2': 'EUR',
                    'decimalscale': '2',
                    'format': 'json'
                }

                total_requests += 1

                start_time = time.time()
                response = session.get(f"{BASE_URL}/getreport", params=params, timeout=600)
                end_time = time.time()

                log(f"[{total_requests}] {country} {impexp} Status={response.status_code} Time={end_time-start_time:.1f}s Batch={len(batch)} Sent={total_sent}")

                if response.status_code == 200:
                    data = response.json()

                    if isinstance(data, list) and len(data) > 0:
                        add_to_batch(data, country, impexp)
                    elif isinstance(data, dict):
                        if 'reportData' in data:
                            add_to_batch(data['reportData'], country, impexp)
                        elif 'data' in data:
                            add_to_batch(data['data'], country, impexp)

            except Exception as e:
                log(f"✗ {str(e)}")
                continue

    flush_batch()

    log(f"\nCOMPLETED: {total_requests} requests, {total_sent} records sent")


def get_updated_data(token, countries):
    """БЛОК 2: Регулярное обновление"""
    log("=" * 60)
    log("BLOCK 2: Checking for updates")
    log("=" * 60)

    last_update = get_last_update_time()
    current_date = datetime.now().strftime('%Y-%m')

    fields = 'reporterCode,tradeFlow,isMirror,year,month,reporterName,reporterDescription,reporterSource,incoterm,partnerCode,partnerName,hsCode,commodityDescriptionOriginal,hs6Code,hs6DescriptionOriginal,value,currency,value2,currency2,quantity1,unit1,quantity2,unit2,price1,priceunit1,price2,priceunit2,port,transport,subdivision'

    trade_details = 'PORT,SUBDIVISION,TRANSPORT,CUSTOMS_REGIME,FOREIGN_PORT,FOREIGN_SUBDIVISION'

    for country in countries:
        for impexp in ['E', 'I']:
            try:
                params = {
                    'token': token,
                    'hscode': '31',
                    'reporter': country,
                    'impexp': impexp,
                    'updatedAfter': last_update if last_update else '2020-01',
                    'latestavailablemonths': 3,
                    'layout': 'brief',
                    'hslevel': '6',
                    'field': fields,
                    'tradedetails': trade_details,
                    'currency2': 'EUR',
                    'decimalscale': '2',
                    'format': 'json'
                }

                response = session.get(f"{BASE_URL}/getreport", params=params, timeout=300)
                if response.status_code == 200:
                    data = response.json()
                    if isinstance(data, list) and len(data) > 0:
                        add_to_batch(data, country, impexp)
                    elif isinstance(data, dict) and 'reportData' in data:
                        add_to_batch(data['reportData'], country, impexp)

            except Exception as e:
                log(f"✗ {str(e)}")
                continue

    flush_batch()
    save_last_update_time(current_date)

    log(f"Updated. Total sent: {total_sent}")


def main():
    """Основная функция"""
    log('--------------------- Script started ------------------------------')

    try:
        token = get_token()
        if not token:
            log("ERROR: Failed to get token")
            sys.exit(1)

        countries = get_subscription_countries(token)

        if not countries:
            log("ERROR: Failed to get parameters")
            sys.exit(1)

        last_update = get_last_update_time()

        if last_update is None:
            log("First run - BLOCK 1: Historical collection")
            get_historical_data(token, countries)
            save_last_update_time(datetime.now().strftime('%Y-%m'))
        else:
            log(f"Regular run - BLOCK 2: Update (last: {last_update})")
            get_updated_data(token, countries)

        log(f"Total sent: {total_sent}")

    except Exception as e:
        log(f'Error: {traceback.format_exception(*sys.exc_info())}')
        sys.exit(1)
    finally:
        log('--------------------- Script finished -----------------------------')


if __name__ == "__main__":
    main()