---

https://www.globaltradetracker.com/api/rest/getreport?impexp=e&hscode=31&from=2026-02&to=2026-02&reporter=TR_national&token=57EC5975835721D871FADA5703FF264B&layout=brief&hslevel=6&field=reporterCode,tradeFlow,isMirror,year,month,reporterName,reporterDescription,reporterSource,incoterm,partnerCode,partnerName,hsCode,commodityDescriptionOriginal,hs6Code,hs6DescriptionOriginal,value,currency,value2,currency2,quantity1,unit1,quantity2,unit2,price1,priceunit1,price2,priceunit2,port,transport,subdivision&tradedetails=PORT,SUBDIVISION,TRANSPORT,CUSTOMS_REGIME,FOREIGN_PORT,FOREIGN_SUBDIVISION&format=json&currency2=EUR&decimalscale=2

нужно переделать чтоб парсило по такого формата ссылке, будет только hscode 31 по всем странам также надо определить токен
а период поменять на 2020 по 2026

import builtins
import json
import os
import sys
import logging
import traceback
import requests
from datetime import datetime, timedelta
from calendar import monthrange
from argparse import ArgumentParser
from io import StringIO
import csv
import time
import gc
from multiprocessing import Process, Queue

from umdap_config import LOG_FOLDER, ENV_FOLDER, EROUTER_HOST, PRIMARY_MADAB, UMDAP_ETL_URL_LIST

APPLICATION_NAME = 'externalGTT'
LOG_FILE = os.path.join(LOG_FOLDER, "externalGTT.log")
CONFIG_FILE = os.path.join(ENV_FOLDER, "externalGtt.json")
CREDENTIALS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.json")

BASE_URL = 'https://www.globaltradetracker.com/api/rest'
builtins.BASE_URL = BASE_URL
builtins.XML_DATE_FMT = '%d.%m.%Y %H:%M:%S'
builtins.DELIMITER = ','

session = requests.Session()
session.headers.update({
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'application/json, text/plain, */*',
    'Accept-Language': 'en-US,en;q=0.9',
    'Accept-Encoding': 'gzip, deflate, br',
    'Connection': 'keep-alive',
})

last_update_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "last_update.json")

BATCH_SIZE = 500000
batch = list()
total_sent = 0

ALL_KEYS = [
    'Reporter_Code', 'Trade_Flow', 'Is_Mirror_Data', 'Period',
    'Reporter_Name', 'Reporter_Description', 'Reporter_Source',
    'Incoterm', 'Partner_Code', 'Partner_Name',
    'Commodity_HS_Code', 'Commodity_Description',
    'HS6_Code', 'HS6_Code_Description',
    'Subdivision', 'Port', 'Transport', 'Foreign_Port', 'US_State',
    'Customs_Regime', 'Suppression',
    'Monetary_Value', 'Currency',
    'Secondary_Monetary_Value', 'Secondary_Currency',
    'Primary_Quantity', 'Primary_Quantity_Unit',
    'Secondary_Quantity', 'Secondary_Quantity_Unit',
    'Primary_Quantity_Price', 'Primary_Quantity_Price_Unit',
    'Secondary_Quantity_Price', 'Secondary_Quantity_Price_Unit'
]


def log(msg=''):
    """Логирование с временной меткой"""
    str_with_time = f'{datetime.now().strftime("%Y/%m/%d %H:%M:%S")} {msg}'
    print(str_with_time)
    with open(LOG_FILE, "a", encoding="utf-8") as file:
        file.write(f'{str_with_time}\n')

builtins.log = log


def get_token():
    """Получение токена для API"""
    try:
        with open(CREDENTIALS_FILE, 'r') as f:
            credentials = json.load(f)

        link = f"{BASE_URL}/gettoken?userid={credentials['user_id']}&password={credentials['password']}"
        response = session.get(link, timeout=30)

        if response.status_code == 200:
            token = response.content.decode().strip()
            if token:
                log("Token received successfully")
                return token
        return None
    except:
        return None


def get_subscription_countries(token):
    """Получение списка стран"""
    try:
        response = session.get(f"{BASE_URL}/countries?token={token}", timeout=30)
        if response.status_code == 200:
            data = response.json()
            countries = []
            if isinstance(data, list):
                for item in data:
                    if isinstance(item, dict):
                        country_data = item.get('country', item)
                        if country_data.get('reportingcountry') == 'true' and country_data.get('defaultReporter') == True:
                            code = country_data.get('reportercode', '')
                            if code:
                                countries.append(code)
            countries = list(dict.fromkeys(countries))
            log(f"Available default reporters ({len(countries)})")
            return countries
        return []
    except:
        return []


def get_subscription_hs_codes(token):
    """Получение списка HS кодов"""
    try:
        response = session.get(f"{BASE_URL}/subscriptions?token={token}", timeout=30)
        if response.status_code == 200:
            data = response.json()
            hs_codes = []
            if isinstance(data, list):
                for item in data:
                    if isinstance(item, dict):
                        codes = item.get('hsCodes', [])
                        if codes:
                            hs_codes.extend([str(code) for code in codes])
            log(f"Available HS codes ({len(hs_codes)}): {hs_codes}")
            return hs_codes
        return []
    except:
        return []


def get_last_update_time():
    """Чтение времени последнего обновления"""
    try:
        if os.path.exists(last_update_file):
            with open(last_update_file, 'r') as f:
                data = json.load(f)
                return data.get('last_update', None)
        return None
    except:
        return None


def save_last_update_time(update_time):
    """Сохранение времени последнего обновления"""
    try:
        with open(last_update_file, 'w') as f:
            json.dump({'last_update': update_time}, f)
    except:
        pass


def parse_period(period_data):
    """Парсинг периода"""
    try:
        if isinstance(period_data, dict):
            year = period_data.get('year', 2000)
            month = period_data.get('month', 1)
        elif isinstance(period_data, list) and len(period_data) >= 2:
            year, month = period_data[0], period_data[1]
        elif isinstance(period_data, str):
            parts = period_data.split('-')
            if len(parts) >= 2:
                year, month = int(parts[0]), int(parts[1])
                if len(parts) == 3:
                    return datetime(year, month, int(parts[2])).strftime('%Y-%m-%d')
            else:
                return '2000-01-01'
        else:
            return '2000-01-01'
        last_day = monthrange(int(year), int(month))[1]
        return datetime(int(year), int(month), last_day).strftime('%Y-%m-%d')
    except:
        return '2000-01-01'


def parse_record_to_dict(record, reporter_code="", trade_flow=""):
    """Парсинг записи"""
    try:
        if not isinstance(record, dict):
            return None

        def safe_get(obj, key, default=''):
            if isinstance(obj, dict):
                return obj.get(key, default)
            return default

        reporter = record.get('reporter', {})
        partner = record.get('partner', {})
        commodity = record.get('commodity', {})

        value = record.get('value', 0)
        if isinstance(value, dict):
            value = value.get('number', 0)

        quantity1 = record.get('quantity1', 0)
        if isinstance(quantity1, dict):
            quantity1 = quantity1.get('number', 0)

        quantity2 = record.get('quantity2', 0)
        if isinstance(quantity2, dict):
            quantity2 = quantity2.get('number', 0)

        return {
            'Reporter_Code': safe_get(reporter, 'code', reporter_code) if isinstance(reporter, dict) else reporter_code,
            'Trade_Flow': record.get('tradeFlow', trade_flow),
            'Is_Mirror_Data': record.get('isMirrorData', ''),
            'Period': parse_period(record.get('period')),
            'Reporter_Name': safe_get(reporter, 'name', ''),
            'Reporter_Description': safe_get(reporter, 'description', ''),
            'Reporter_Source': safe_get(reporter, 'source', ''),
            'Incoterm': record.get('incoterm', ''),
            'Partner_Code': safe_get(partner, 'code', ''),
            'Partner_Name': safe_get(partner, 'name', ''),
            'Commodity_HS_Code': safe_get(commodity, 'hsCode', ''),
            'Commodity_Description': safe_get(commodity, 'description', ''),
            'HS6_Code': record.get('hs6Code', ''),
            'HS6_Code_Description': record.get('hs6Description', ''),
            'Subdivision': record.get('subdivision', ''),
            'Port': record.get('port', ''),
            'Transport': record.get('transport', ''),
            'Foreign_Port': record.get('foreignPort', ''),
            'US_State': record.get('usState', ''),
            'Customs_Regime': record.get('customsRegime', ''),
            'Suppression': record.get('suppression', ''),
            'Monetary_Value': value,
            'Currency': record.get('currency', 'USD'),
            'Secondary_Monetary_Value': record.get('secondaryValue', 0),
            'Secondary_Currency': record.get('secondaryCurrency', ''),
            'Primary_Quantity': quantity1,
            'Primary_Quantity_Unit': record.get('quantity1Unit', ''),
            'Secondary_Quantity': quantity2,
            'Secondary_Quantity_Unit': record.get('quantity2Unit', ''),
            'Primary_Quantity_Price': record.get('price1', 0),
            'Primary_Quantity_Price_Unit': record.get('price1Unit', ''),
            'Secondary_Quantity_Price': record.get('price2', 0),
            'Secondary_Quantity_Price_Unit': record.get('price2Unit', ''),
        }
    except:
        return None


def send_batch_in_subprocess(batch_data, queue):
    """Отправка батча в отдельном процессе"""
    temp_file = f'temp_batch_{os.getpid()}.csv'

    try:
        with open(temp_file, 'w', newline='', encoding='utf-8') as f:
            writer = csv.DictWriter(f, fieldnames=ALL_KEYS)
            writer.writeheader()
            writer.writerows(batch_data)

        from etl_common import process_etls

        def batch_request(url):
            with open(temp_file, 'r') as f:
                return f.read()

        process_etls(
            application_name=APPLICATION_NAME,
            etl_desc_url=f'{EROUTER_HOST}/procedure/{PRIMARY_MADAB}/umdap.etl.description/call.json?application={APPLICATION_NAME}',
            data_type=None,
            api_requestor=batch_request,
            erouter_url_list=UMDAP_ETL_URL_LIST
        )

        queue.put(('success', len(batch_data)))
    except Exception as e:
        queue.put(('error', str(e)))
    finally:
        if os.path.exists(temp_file):
            os.remove(temp_file)


def send_batch_to_dwh(batch_data):
    """Отправка батча в DWH через отдельный процесс"""
    global total_sent, batch

    if not batch_data:
        return

    log(f"Sending batch of {len(batch_data)} records...")

    queue = Queue()
    p = Process(target=send_batch_in_subprocess, args=(batch_data, queue))
    p.start()

    p.join(timeout=600)

    if p.is_alive():
        log("✗ Timeout! Terminating...")
        p.terminate()
        p.join()
    else:
        if not queue.empty():
            status, result = queue.get()
            if status == 'success':
                total_sent += result
                log(f"✓ Batch sent. Total sent: {total_sent}")
            else:
                log(f"✗ Error: {result}")

    # Очистка
    batch_data = None
    batch = []
    gc.collect()


def add_to_batch(records, reporter_code="", trade_flow=""):
    """Добавление записей в порцию"""
    global batch

    for record in records:
        if isinstance(record, dict):
            parsed = parse_record_to_dict(record, reporter_code, trade_flow)
            if parsed:
                batch.append(parsed)
                if len(batch) >= BATCH_SIZE:
                    send_batch_to_dwh(batch)


def flush_batch():
    """Отправка оставшихся"""
    if batch:
        send_batch_to_dwh(batch)


def check_data_updates(token):
    """Проверка обновлений"""
    try:
        last_update = get_last_update_time()
        if not last_update:
            return True

        link = f"{BASE_URL}/dataupdates?token={token}&updatedAfter={last_update}"
        response = session.get(link, timeout=30)
        if response.status_code == 200:
            data = response.json()
            return data and len(data) > 0
        return False
    except:
        return False


def get_historical_data(token, countries, hs_codes):
    """БЛОК 1: Исторический сбор с 2013"""
    log("=" * 60)
    log("BLOCK 1: Starting historical data collection from 2013")
    log(f"Countries ({len(countries)}), HS ({len(hs_codes)})")
    log("=" * 60)

    current_year = datetime.now().year
    current_month = datetime.now().month
    all_hs = ','.join(hs_codes)

    total_requests = 0

    for country in countries:
        for impexp in ['E', 'I']:
            try:
                params = {
                    'token': token,
                    'hscode': all_hs,
                    'reporter': country,
                    'impexp': impexp,
                    'from': '2013-01',
                    'to': f'{current_year}-{current_month:02d}',
                    'currency': 'USD',
                    'format': 'json'
                }

                total_requests += 1

                start_time = time.time()
                response = session.get(f"{BASE_URL}/getreport", params=params, timeout=600)
                end_time = time.time()

                log(f"[{total_requests}] {country} {impexp} Status={response.status_code} Time={end_time-start_time:.1f}s Batch={len(batch)} Sent={total_sent}")

                if response.status_code == 200:
                    data = response.json()
                    if isinstance(data, list) and len(data) > 0:
                        add_to_batch(data, country, impexp)
                    elif isinstance(data, dict) and 'data' in data:
                        add_to_batch(data['data'], country, impexp)

            except Exception as e:
                log(f"✗ {str(e)}")
                continue

    flush_batch()

    log(f"\nCOMPLETED: {total_requests} requests, {total_sent} records sent")


def get_updated_data(token, countries, hs_codes):
    """БЛОК 2: Регулярное обновление"""
    log("=" * 60)
    log("BLOCK 2: Checking for updates")
    log("=" * 60)

    last_update = get_last_update_time()
    current_date = datetime.now().strftime('%Y-%m')

    if not check_data_updates(token):
        log("No updates available")
        return

    all_hs = ','.join(hs_codes)

    for country in countries:
        for impexp in ['E', 'I']:
            try:
                params = {
                    'token': token,
                    'hscode': all_hs,
                    'reporter': country,
                    'impexp': impexp,
                    'updatedAfter': last_update if last_update else '2013-01',
                    'latestavailablemonths': 3,
                    'currency': 'USD',
                    'format': 'json'
                }

                response = session.get(f"{BASE_URL}/getreport", params=params, timeout=300)
                if response.status_code == 200:
                    data = response.json()
                    if isinstance(data, list) and len(data) > 0:
                        add_to_batch(data, country, impexp)
                    elif isinstance(data, dict) and 'data' in data:
                        add_to_batch(data['data'], country, impexp)

            except Exception as e:
                log(f"✗ {str(e)}")
                continue

    flush_batch()
    save_last_update_time(current_date)

    log(f"Updated. Total sent: {total_sent}")


def main():
    """Основная функция"""
    log('--------------------- Script started ------------------------------')

    try:
        token = get_token()
        if not token:
            log("ERROR: Failed to get token")
            sys.exit(1)

        countries = get_subscription_countries(token)
        hs_codes = get_subscription_hs_codes(token)

        if not countries or not hs_codes:
            log("ERROR: Failed to get parameters")
            sys.exit(1)

        last_update = get_last_update_time()

        if last_update is None:
            # БЛОК 1: Первый запуск
            log("First run - BLOCK 1: Historical collection")
            get_historical_data(token, countries, hs_codes)
            save_last_update_time(datetime.now().strftime('%Y-%m'))
        else:
            # БЛОК 2: Обновление
            log(f"Regular run - BLOCK 2: Update (last: {last_update})")
            get_updated_data(token, countries, hs_codes)

        log(f"Total sent: {total_sent}")

    except Exception as e:
        log(f'Error: {traceback.format_exception(*sys.exc_info())}')
        sys.exit(1)
    finally:
        log('--------------------- Script finished -----------------------------')


if __name__ == "__main__":
    main()