<?php
/**
* @version	$Id: cat_dbitem_export_helper.php 16692 2021-08-31 09:24:09Z alex $
* @package	In-Portal
* @copyright	Copyright (C) 1997 - 2009 Intechnic. All rights reserved.
* @license      GNU/GPL
* In-Portal is Open Source software.
* This means that this software may have been modified pursuant
* the GNU General Public License, and as distributed it includes
* or is derivative of works licensed under the GNU General Public License
* or other free or open source software licenses.
* See http://www.in-portal.org/license for copyright notices and details.
*/

	defined('FULL_PATH') or die('restricted access!');

	define('EXPORT_STEP', 100); // export by 200 items (e.g. links)
	define('IMPORT_STEP', 20); // export by 200 items (e.g. links)
	define('IMPORT_CHUNK', 10240); // 10240); //30720); //50120); // 5 KB

	define('IMPORT_TEMP', 1);
	define('IMPORT_LIVE', 2);

	class kCatDBItemExportHelper extends kHelper {

		var $false = false;

		var $cache = Array();

		/**
		 * Allows to find out what items are new in cache
		 *
		 * @var Array
		 */
		var $cacheStatus = Array();

		var $cacheTable = '';

		var $exportFields = Array();

		/**
		 * Export options
		 *
		 * @var Array
		 */
		var $exportOptions = Array();

		/**
		 * Item beeing currenly exported
		 *
		 * @var kCatDBItem
		 */
		var $curItem = null;

		/**
		 * Dummy category object
		 *
		 * @var CategoriesItem
		 */
		var $dummyCategory = null;

		/**
		 * Pointer to opened file
		 *
		 * @var resource
		 */
		var $filePointer = null;

		/**
		 * Custom fields definition of current item
		 *
		 * @var Array
		 */
		var $customFields = Array();

		public function __construct()
		{
			parent::__construct();

			$this->cacheTable = TABLE_PREFIX.'ImportCache';
		}

		/**
		 * Returns value from cache if found or false otherwise
		 *
		 * @param string $type
		 * @param int $key
		 * @return mixed
		 */
		function getFromCache($type, $key)
		{
			return getArrayValue($this->cache, $type, $key);
		}

		/**
		 * Adds value to be cached
		 *
		 * @param string $type
		 * @param int $key
		 * @param mixed $value
		 * @param bool $is_new
		 */
		function addToCache($type, $key, $value, $is_new = true)
		{
			/*if ( !isset($this->cache[$type]) ) {
				$this->cache[$type] = Array ();
			}*/

			$this->cache[$type][$key] = $value;

			if ( $is_new ) {
				$this->cacheStatus[$type][$key] = true;
			}
		}

		function storeCache($cache_types)
		{
			$fields_hash = Array ();
			$cache_types = explode(',', $cache_types);

			foreach ($cache_types as $cache_type) {
				$fields_hash = Array ('CacheName' => $cache_type);
				$cache = getArrayValue($this->cacheStatus, $cache_type);

				if ( !$cache ) {
					$cache = Array ();
				}

				foreach ($cache as $var_name => $cache_status) {
					$fields_hash['VarName'] = $var_name;
					$fields_hash['VarValue'] = $this->cache[$cache_type][$var_name];

					$this->Conn->doInsert($fields_hash, $this->cacheTable, 'INSERT', false);
				}
			}

			if ( isset($fields_hash['VarName']) ) {
				$this->Conn->doInsert($fields_hash, $this->cacheTable, 'INSERT');
			}
		}

		function loadCache()
		{
			$this->cache = Array ();

			$sql = 'SELECT *
					FROM ' . $this->cacheTable;
			$records = $this->Conn->GetIterator($sql);

			foreach ($records as $record) {
				$this->addToCache($record['CacheName'], $record['VarName'], $record['VarValue'], false);
			}
		}

		/**
		 * Fill required fields with dummy values
		 *
		 * @param kEvent|bool $event
		 * @param kCatDBItem|bool $object
		 * @param bool $set_status
		 */
		function fillRequiredFields($event, &$object, $set_status = false)
		{
			if ( $object == $this->false ) {
				/** @var kCatDBItem $object */
				$object = $event->getObject();
			}

			$has_empty = false;
			$fields = $object->getFields();

			if ( $object->isField('CreatedById') ) {
				// CSV file was created without required CreatedById column
				if ( $object->isRequired('CreatedById') ) {
					$object->setRequired('CreatedById', false);
				}

				if ( !is_numeric( $object->GetDBField('CreatedById') ) ) {
					$object->SetDBField('CreatedById', $this->Application->RecallVar('user_id'));
				}
			}

			foreach ($fields as $field_name => $field_options) {
				if ( $object->isVirtualField($field_name) || !$object->isRequired($field_name) ) {
					continue;
				}

				if ( $object->GetDBField($field_name) ) {
					continue;
				}

				$formatter_class = getArrayValue($field_options, 'formatter');

				if ( $formatter_class ) {
					// not tested
					/** @var kFormatter $formatter */
					$formatter = $this->Application->recallObject($formatter_class);

					$sample_value = $formatter->GetSample($field_name, $field_options, $object);
				}

				$has_empty = true;
				$object->SetField($field_name, isset($sample_value) && $sample_value ? $sample_value : 'no value');
			}
			$object->UpdateFormattersSubFields();

			if ( $set_status && $has_empty ) {
				$object->SetDBField('Status', 0);
			}
		}

		/**
		 * Verifies that all user entered export params are correct
		 *
		 * @param kEvent $event
		 * @return bool
		 * @access protected
		 */
		protected function verifyOptions($event)
		{
			if ($this->Application->RecallVar($event->getPrefixSpecial().'_ForceNotValid'))
			{
				$this->Application->StoreVar($event->getPrefixSpecial().'_ForceNotValid', 0);
				return false;
			}

			$this->fillRequiredFields($event, $this->false);

			/** @var kCatDBItem $object */
			$object = $event->getObject();

			$cross_unique_fields = Array('FieldsSeparatedBy', 'FieldsEnclosedBy');
			if (($object->GetDBField('CategoryFormat') == 1) || ($event->Special == 'import')) // in one field
			{
				$object->setRequired('CategorySeparator');
				$cross_unique_fields[] = 'CategorySeparator';
			}

			$ret = $object->Validate();

			// check if cross unique fields has no same values
			foreach ($cross_unique_fields as $field_index => $field_name)
			{
				if ($object->GetErrorPseudo($field_name) == 'required') {
					continue;
				}

				$check_fields = $cross_unique_fields;
				unset($check_fields[$field_index]);

				foreach ($check_fields as $check_field)
				{
					if ($object->GetDBField($field_name) == $object->GetDBField($check_field))
					{
						$object->SetError($check_field, 'unique');
					}
				}
			}

			if ($event->Special == 'import')
			{
				$this->exportOptions = $this->loadOptions($event);

				$automatic_fields = ($object->GetDBField('FieldTitles') == 1);
				$object->setRequired('ExportColumns', !$automatic_fields);
				$category_prefix = '__CATEGORY__';
				if ( $automatic_fields && ($this->exportOptions['SkipFirstRow']) ) {
					$this->openFile($event);
					$this->exportOptions['ExportColumns'] = $this->readRecord();

					if (!$this->exportOptions['ExportColumns']) {
						$this->exportOptions['ExportColumns'] = Array ();
					}

					$this->closeFile();

					// remove additional (non-parseble columns)
					foreach ($this->exportOptions['ExportColumns'] as $field_index => $field_name) {
						if (!$this->validateField($field_name, $object)) {
							unset($this->exportOptions['ExportColumns'][$field_index]);
						}
					}
					$category_prefix = '';
				}

				// 1. check, that we have column definitions
				if (!$this->exportOptions['ExportColumns']) {
					$object->setError('ExportColumns', 'required');
					$ret = false;
				}
				else {
					// 1.1. check that all required fields are present in imported file
					$missing_columns = Array();
					$fields = $object->getFields();

					foreach ($fields as $field_name => $field_options) {
						if ($object->skipField($field_name)) continue;
						if ( $object->isRequired($field_name) && !in_array($field_name, $this->exportOptions['ExportColumns']) ) {
							$missing_columns[] = $field_name;
							$object->setError('ExportColumns', 'required_fields_missing', 'la_error_RequiredColumnsMissing');
							$ret = false;
						}
					}

					if (!$ret && $this->Application->isDebugMode()) {
						$this->Application->Debugger->appendHTML('Missing required for import/export:');
						$this->Application->Debugger->dumpVars($missing_columns);
					}
				}


				// 2. check, that we have only mixed category field or only separated category fields
				$category_found['mixed'] = false;
				$category_found['separated'] = false;

				foreach ($this->exportOptions['ExportColumns'] as $import_field) {
					if (preg_match('/^'.$category_prefix.'Category(Path|[0-9]+)/', $import_field, $rets)) {
						$category_found[$rets[1] == 'Path' ? 'mixed' : 'separated'] = true;
					}
				}
				if ($category_found['mixed'] && $category_found['separated']) {
					$object->SetError('ExportColumns', 'unique_category', 'la_error_unique_category_field');
					$ret = false;
				}

				// 3. check, that duplicates check fields are selected & present in imported fields
				if ($this->exportOptions['ReplaceDuplicates']) {
					if ($this->exportOptions['CheckDuplicatesMethod'] == 1) {
						$check_fields = Array($object->IDField);
					}
					else {
						$check_fields = $this->exportOptions['DuplicateCheckFields'] ? explode('|', substr($this->exportOptions['DuplicateCheckFields'], 1, -1)) : Array();
						$object = $event->getObject();

						$fields = $object->getFields();
						$language_id = $this->Application->GetDefaultLanguageId();

						foreach ($check_fields as $index => $check_field) {
							foreach ($fields as $field_name => $field_options) {
								if ($field_name == 'l'.$language_id.'_'.$check_field) {
									$check_fields[$index] = 'l'.$language_id.'_'.$check_field;
									break;
								}
							}
						}
					}
					$this->exportOptions['DuplicateCheckFields'] = $check_fields;

					if (!$check_fields) {
						$object->setError('CheckDuplicatesMethod', 'required');
						$ret = false;
					}
					else {
						foreach ($check_fields as $check_field) {
							$check_field = preg_replace('/^cust_(.*)/', 'Custom_\\1', $check_field);
							if (!in_array($check_field, $this->exportOptions['ExportColumns'])) {
								$object->setError('ExportColumns', 'required');
								$ret = false;
								break;
							}
						}
					}
				}
				$this->saveOptions($event);
			}

			return $ret;
		}

		/**
		 * Returns filename to read import data from
		 *
		 * @return string
		 */
		function getImportFilename()
		{
			if ($this->exportOptions['ImportSource'] == 1)
			{
				$ret = $this->exportOptions['ImportFilename']; // ['name']; commented by Kostja
			}
			else {
				$ret = $this->exportOptions['ImportLocalFilename'];
			}
			return EXPORT_PATH.'/'.$ret;
		}

		/**
		 * Returns filename to write export data to
		 *
		 * @return string
		 */
		function getExportFilename()
		{
			$extension = $this->getFileExtension();
			$filename = preg_replace('/(.*)\.' . $extension . '$/', '\1', $this->exportOptions['ExportFilename']) . '.' . $extension;

			return EXPORT_PATH . DIRECTORY_SEPARATOR . $filename;
		}

		/**
		 * Opens file required for export/import operations
		 *
		 * @param kEvent $event
		 */
		function openFile($event)
		{
			/** @var FileHelper $file_helper */
			$file_helper = $this->Application->recallObject('FileHelper');

			$file_helper->CheckFolder(EXPORT_PATH);

			if ( $event->Special == 'export' ) {
				$first_step = $this->exportOptions['start_from'] == 0;
				$this->filePointer = fopen($this->getExportFilename(), $first_step ? 'w' : 'r+');

				if ( !$first_step ) {
					fseek($this->filePointer, 0, SEEK_END);
				}
			}
			else {
				$this->filePointer = fopen($this->getImportFilename(), 'r');

				// skip UTF-8 BOM Modifier
				$first_chars = fread($this->filePointer, 3);
				if ( bin2hex($first_chars) != 'efbbbf' ) {
					fseek($this->filePointer, 0);
				}
			}
		}

		/**
		 * Closes opened file
		 *
		 */
		function closeFile()
		{
			fclose($this->filePointer);
		}

		function getCustomSQL()
		{
			/** @var kMultiLanguage $ml_formatter */
			$ml_formatter = $this->Application->recallObject('kMultiLanguage');

			$custom_sql = '';

			foreach ($this->customFields as $custom_id => $custom_name) {
				$custom_sql .= 'custom_data.' . $ml_formatter->LangFieldName('cust_' . $custom_id) . ' AS cust_' . $custom_name . ', ';
			}

			return substr($custom_sql, 0, -2);
		}

		function getPlainExportSQL($count_only = false)
		{
			if ( $count_only && isset($this->exportOptions['ForceCountSQL']) ) {
				$sql = $this->exportOptions['ForceCountSQL'];
			}
			elseif ( !$count_only && isset($this->exportOptions['ForceSelectSQL']) ) {
				$sql = $this->exportOptions['ForceSelectSQL'];
			}
			else {
				/** @var kDBList $items_list */
				$items_list = $this->Application->recallObject(
					$this->curItem->Prefix . '.' . $this->exportOptions['export_special'],
					$this->curItem->Prefix . '_List',
					array('grid' => $this->exportOptions['export_grid'])
				);

				$items_list->SetPerPage(-1);

				if ( $this->exportOptions['export_ids'] != '' ) {
					$items_list->addFilter('export_ids', $items_list->TableName . '.' . $items_list->IDField . ' IN (' . implode(',', $this->exportOptions['export_ids']) . ')');
				}

				if ( $count_only ) {
					$sql = $items_list->getCountSQL($items_list->GetSelectSQL(true, false));
				}
				else {
					$sql = $items_list->GetSelectSQL();
				}
			}

			if ( !$count_only ) {
				$sql .= ' LIMIT ' . $this->exportOptions['start_from'] . ',' . EXPORT_STEP;
			}
			/*else {
				$sql = preg_replace("/^\s*SELECT(.*?\s)FROM(?!_)/is", "SELECT COUNT(*) AS count FROM ", $sql);
			}*/

			return $sql;
		}

		function getExportSQL($count_only = false)
		{
			if ( !$this->Application->getUnitOption($this->curItem->Prefix, 'CatalogItem') ) {
				return $this->GetPlainExportSQL($count_only); // in case this is not a CategoryItem
			}

			if ( $this->exportOptions['export_ids'] === false ) {
				// get links from current category & all it's subcategories
				$join_clauses = Array ();

				$custom_sql = $this->getCustomSQL();

				if ( $custom_sql ) {
					$custom_table = $this->Application->getUnitOption($this->curItem->Prefix . '-cdata', 'TableName');
					$join_clauses[$custom_table . ' custom_data'] = 'custom_data.ResourceId = item_table.ResourceId';
				}

				$join_clauses[TABLE_PREFIX . 'CategoryItems ci'] = 'ci.ItemResourceId = item_table.ResourceId';
				$join_clauses[TABLE_PREFIX . 'Categories c'] = 'c.CategoryId = ci.CategoryId';

				$sql = 'SELECT item_table.*, ci.CategoryId' . ($custom_sql ? ', ' . $custom_sql : '') . '
						FROM ' . $this->curItem->TableName . ' item_table';

				foreach ($join_clauses as $table_name => $join_expression) {
					$sql .= ' LEFT JOIN ' . $table_name . ' ON ' . $join_expression;
				}

				$sql .= ' WHERE ';

				if ( $this->exportOptions['export_cats_ids'][0] == 0 ) {
					$sql .= '1';
				}
				else {
					foreach ($this->exportOptions['export_cats_ids'] as $category_id) {
						$sql .= '(c.ParentPath LIKE "%|' . $category_id . '|%") OR ';
					}

					$sql = substr($sql, 0, -4);
				}

				$sql .= ' ORDER BY ci.PrimaryCat DESC, c.TreeLeft ASC, item_table.' . $this->curItem->IDField . ' ASC';
			}
			else {
				// get only selected links
				$sql = 'SELECT item_table.*, ' . $this->exportOptions['export_cats_ids'][0] . ' AS CategoryId
						FROM ' . $this->curItem->TableName . ' item_table
						WHERE ' . $this->curItem->IDField . ' IN (' . implode(',', $this->exportOptions['export_ids']) . ')';
			}

			if ( !$count_only ) {
				$sql .= ' LIMIT ' . $this->exportOptions['start_from'] . ',' . EXPORT_STEP;
			}
			else {
				$sql = preg_replace("/^\s*SELECT(.*?\s)FROM(?!_)/is", "SELECT COUNT(*) AS count FROM ", $sql);
			}

			return $sql;
		}

		/**
		 * Enter description here...
		 *
		 * @param kEvent $event
		 */
		function performExport($event)
		{
			$this->exportOptions = $this->loadOptions($event);
			$this->exportFields = $this->exportOptions['ExportColumns'];
			$this->curItem = $event->getObject( Array('skip_autoload' => true) );
			$this->customFields = $this->Application->getUnitOption($event->Prefix, 'CustomFields');
			$this->openFile($event);

			if ($this->exportOptions['start_from'] == 0) // first export step
			{
				if (!getArrayValue($this->exportOptions, 'IsBaseCategory')) {
					$this->exportOptions['IsBaseCategory'] = 0;
				}

				if ($this->exportOptions['IsBaseCategory'] ) {
					$sql = 'SELECT ParentPath
							FROM '.TABLE_PREFIX.'Categories
							WHERE CategoryId = ' . (int)$this->Application->GetVar('m_cat_id');
					$parent_path = $this->Conn->GetOne($sql);
					$parent_path = explode('|', substr($parent_path, 1, -1));
					if ($parent_path && $parent_path[0] == $this->Application->getBaseCategory()) {
						array_shift($parent_path);
					}

					$this->exportOptions['BaseLevel'] = count($parent_path); // level to cut from other categories
				}

				// 1. export field titles if required
				if ($this->exportOptions['IncludeFieldTitles'])
				{
					$data_array = Array();
					foreach ($this->exportFields as $export_field)
					{
						$data_array = array_merge($data_array, $this->getFieldCaption($export_field));
					}
					$this->writeRecord($data_array);
				}
				$this->exportOptions['total_records'] = $this->Conn->GetOne( $this->getExportSQL(true) );
			}

			// 2. export data
			$records = $this->Conn->Query( $this->getExportSQL() );
			$records_exported = 0;
			foreach ($records as $record_info) {
				$this->curItem->LoadFromHash($record_info);

				$data_array = Array();
				foreach ($this->exportFields as $export_field)
				{
					$data_array = array_merge($data_array, $this->getFieldValue($export_field) );
				}
				$this->writeRecord($data_array);
				$records_exported++;
			}
			$this->closeFile();

			$this->exportOptions['start_from'] += $records_exported;
			$this->saveOptions($event);

			return $this->exportOptions;
		}

		function getItemFields()
		{
			// just in case dummy user selected automtic mode & moved columns too :(
			$src_options = $this->curItem->GetFieldOption('ExportColumns', 'options');
			$dst_options = $this->curItem->GetFieldOption('AvailableColumns', 'options');

			return array_merge($dst_options, $src_options);
		}

		/**
		 * Checks if field really belongs to importable field list
		 *
		 * @param string $field_name
		 * @param kCatDBItem $object
		 * @return bool
		 */
		function validateField($field_name, &$object)
		{
			// 1. convert custom field
			$field_name = preg_replace('/^Custom_(.*)/', '__CUSTOM__\\1', $field_name);

			// 2. convert category field (mixed version & separated version)
			$field_name = preg_replace('/^Category(Path|[0-9]+)/', '__CATEGORY__Category\\1', $field_name);

			$valid_fields = $object->getPossibleExportColumns();
			return isset($valid_fields[$field_name]) || isset($valid_fields['__VIRTUAL__'.$field_name]);
		}

		/**
		 * Enter description here...
		 *
		 * @param kEvent $event
		 */
		function performImport($event)
		{
			if (!$this->exportOptions) {
				// load import options in case if not previously loaded in verification function
				$this->exportOptions = $this->loadOptions($event);
			}

			$backup_category_id = $this->Application->GetVar('m_cat_id');
			$this->Application->SetVar('m_cat_id', (int)$this->Application->RecallVar('ImportCategory') );

			$this->openFile($event);

			$bytes_imported = 0;
			if ($this->exportOptions['start_from'] == 0) // first export step
			{
				// 1st time run
				if ($this->exportOptions['SkipFirstRow']) {
					$this->readRecord();
					$this->exportOptions['start_from'] = ftell($this->filePointer);
					$bytes_imported = ftell($this->filePointer);
				}

				$current_category_id = $this->Application->GetVar('m_cat_id');
				if ($current_category_id > 0) {
					$sql = 'SELECT ParentPath FROM '.TABLE_PREFIX.'Categories WHERE CategoryId = '.$current_category_id;
					$this->exportOptions['ImportCategoryPath'] = $this->Conn->GetOne($sql);
				}
				else {
					$this->exportOptions['ImportCategoryPath'] = '';
				}
				$this->exportOptions['total_records'] = filesize($this->getImportFilename());
			}
			else {
				$this->loadCache();
			}

			$this->exportFields = $this->exportOptions['ExportColumns'];
			$this->addToCache('category_parent_path', $this->Application->GetVar('m_cat_id'), $this->exportOptions['ImportCategoryPath']);

			// 2. import data
			$this->dummyCategory = $this->Application->recallObject('c.-tmpitem', 'c', Array('skip_autoload' => true));
			fseek($this->filePointer, $this->exportOptions['start_from']);

			$items_processed = 0;
			while (($bytes_imported < IMPORT_CHUNK && $items_processed < IMPORT_STEP) && !feof($this->filePointer)) {
				$data = $this->readRecord();
				if ($data) {
					if ($this->exportOptions['ReplaceDuplicates']) {
						// set fields used as keys for replace duplicates code
						$this->resetImportObject($event, IMPORT_TEMP, $data);
					}

					$this->processCurrentItem($event, $data);
				}
				$bytes_imported = ftell($this->filePointer) - $this->exportOptions['start_from'];
				$items_processed++;
			}

			$this->closeFile();
			$this->Application->SetVar('m_cat_id', $backup_category_id);

			$this->exportOptions['start_from'] += $bytes_imported;
			$this->storeCache('new_ids');

			$this->saveOptions($event);

			if ($this->exportOptions['start_from'] == $this->exportOptions['total_records']) {
				$this->Conn->Query('TRUNCATE TABLE '.$this->cacheTable);
			}

			return $this->exportOptions;
		}

		function setCurrentID()
		{
			$this->curItem->setID( $this->curItem->GetDBField($this->curItem->IDField) );
		}

		/**
		 * Sets value of import/export object
		 * @param int $field_index
		 * @param mixed $value
		 * @return void
		 * @access protected
		 */
		protected function setFieldValue($field_index, $value)
		{
			if ( empty($value) ) {
				$value = null;
			}

			$field_name = getArrayValue($this->exportFields, $field_index);
			if ( $field_name == 'ResourceId' ) {
				return ;
			}

			if ( substr($field_name, 0, 7) == 'Custom_' ) {
				$field_name = 'cust_' . substr($field_name, 7);
				$this->curItem->SetField($field_name, $value);
			}
			elseif ( $field_name == 'CategoryPath' || $field_name == '__CATEGORY__CategoryPath' ) {
				$this->curItem->CategoryPath = $value ? explode($this->exportOptions['CategorySeparator'], $value) : Array ();
			}
			elseif ( substr($field_name, 0, 8) == 'Category' ) {
				$this->curItem->CategoryPath[(int)substr($field_name, 8) - 1] = $value;
			}
			elseif ( substr($field_name, 0, 20) == '__CATEGORY__Category' ) {
				$this->curItem->CategoryPath[(int)substr($field_name, 20) - 1] = $value;
			}
			elseif ( substr($field_name, 0, 11) == '__VIRTUAL__' ) {
				$field_name = substr($field_name, 11);
				$this->curItem->SetField($field_name, $value);
			}
			else {
				$this->curItem->SetField($field_name, $value);
			}

			if ( $this->curItem->GetErrorPseudo($field_name) ) {
				$this->curItem->SetDBField($field_name, null);
				$this->curItem->RemoveError($field_name);
			}
		}

		/**
		 * Resets import object
		 *
		 * @param kEvent $event
		 * @param int $object_type
		 * @param Array $record_data
		 * @return void
		 */
		function resetImportObject($event, $object_type, $record_data = null)
		{
			switch ($object_type) {
				case IMPORT_TEMP:
					$this->curItem = $event->getObject( Array('skip_autoload' => true) );
					break;

				case IMPORT_LIVE:
					$this->curItem = $this->Application->recallObject($event->Prefix.'.-tmpitem'.$event->Special, $event->Prefix, Array('skip_autoload' => true));
					break;
			}
			$this->curItem->Clear();
			$this->curItem->SetDBField('CategoryId', NULL); // since default value is import root category
			$this->customFields = $this->Application->getUnitOption($event->Prefix, 'CustomFields');

			if (isset($record_data)) {
				$this->setImportData($record_data);
			}
		}

		function setImportData($record_data)
		{
			foreach ($record_data as $field_index => $field_value) {
				$this->setFieldValue($field_index, $field_value);
			}
			$this->setCurrentID();
		}


		function getItemCategory()
		{
			static $lang_prefix = null;
			$backup_category_id = $this->Application->GetVar('m_cat_id');

			$category_id = $this->getFromCache('category_names', implode(':', $this->curItem->CategoryPath));
			if ($category_id) {
				$this->Application->SetVar('m_cat_id', $category_id);
				return $category_id;
			}

			if (is_null($lang_prefix)) {
				$lang_prefix = 'l'.$this->Application->GetVar('m_lang').'_';
			}

			foreach ($this->curItem->CategoryPath as $category_index => $category_name) {
				if (!$category_name) continue;
				$category_key = kUtil::crc32( implode(':', array_slice($this->curItem->CategoryPath, 0, $category_index + 1) ) );

				$category_id = $this->getFromCache('category_names', $category_key);
				if ($category_id === false) {
					// get parent category path to search only in it
					$current_category_id = $this->Application->GetVar('m_cat_id');
//					$parent_path = $this->getParentPath($current_category_id);

					// get category id from database by name
					$sql = 'SELECT CategoryId
							FROM '.TABLE_PREFIX.'Categories
							WHERE ('.$lang_prefix.'Name = '.$this->Conn->qstr($category_name).') AND (ParentId = '.(int)$current_category_id.')';
					$category_id = $this->Conn->GetOne($sql);

					if ( $category_id === false ) {
						// category not in db -> create
						$category_fields = Array (
							$lang_prefix.'Name' => $category_name, $lang_prefix.'Description' => $category_name,
							'Status' => STATUS_ACTIVE, 'ParentId' => $current_category_id, 'AutomaticFilename' => 1
						);

						$this->dummyCategory->Clear();
						$this->dummyCategory->SetDBFieldsFromHash($category_fields);

						if ( $this->dummyCategory->Create() ) {
							$category_id = $this->dummyCategory->GetID();
							$this->addToCache('category_parent_path', $category_id, $this->dummyCategory->GetDBField('ParentPath'));
							$this->addToCache('category_names', $category_key, $category_id);
						}
					}
					else {
						$this->addToCache('category_names', $category_key, $category_id);
					}
				}

				if ($category_id) {
					$this->Application->SetVar('m_cat_id', $category_id);
				}
			}
			if (!$this->curItem->CategoryPath) {
				$category_id = $backup_category_id;
			}

			return $category_id;
		}

		/**
		 * Enter description here...
		 *
		 * @param kEvent $event
		 * @param Array $record_data
		 * @return bool
		 */
		function processCurrentItem($event, $record_data)
		{
			$save_method = 'Create';
			$load_keys = Array();

			// create/update categories
			$backup_category_id = $this->Application->GetVar('m_cat_id');

			// perform replace duplicates code
			if ($this->exportOptions['ReplaceDuplicates']) {
				// get replace keys first, then reset current item to empty one
				$category_id = $this->getItemCategory();
				if ($this->exportOptions['CheckDuplicatesMethod'] == 1) {
					if ($this->curItem->GetID()) {
						$load_keys = Array($this->curItem->IDField => $this->curItem->GetID());
					}
				}
				else {
					$key_fields = $this->exportOptions['DuplicateCheckFields'];
					foreach ($key_fields as $key_field) {
						$load_keys[$key_field] = $this->curItem->GetDBField($key_field);
					}
				}

				$this->resetImportObject($event, IMPORT_LIVE);

				if (count($load_keys)) {
					$where_clause = '';
					$language_id = (int)$this->Application->GetVar('m_lang');

					if (!$language_id) {
						$language_id = 1;
					}

					foreach ($load_keys as $field_name => $field_value) {
						if (preg_match('/^cust_(.*)/', $field_name, $regs)) {
							$custom_id = array_search($regs[1], $this->customFields);
							$field_name = 'l'.$language_id.'_cust_'.$custom_id;
							$where_clause .= '(custom_data.`'.$field_name.'` = '.$this->Conn->qstr($field_value).') AND ';
						}
						else {
							$where_clause .= '(item_table.`'.$field_name.'` = '.$this->Conn->qstr($field_value).') AND ';
						}

					}
					$where_clause = substr($where_clause, 0, -5);

					$item_id = $this->getFromCache('new_ids', kUtil::crc32($where_clause));
					if (!$item_id) {
						if ($this->exportOptions['CheckDuplicatesMethod'] == 2) {
							// by other fields
							$parent_path = $this->getParentPath($category_id);
							$where_clause = '(c.ParentPath LIKE "'.$parent_path.'%") AND '.$where_clause;
						}

						$cdata_table = $this->Application->getUnitOption($event->Prefix.'-cdata', 'TableName');
						$sql = 'SELECT '.$this->curItem->IDField.'
								FROM '.$this->curItem->TableName.' item_table
								LEFT JOIN '.$cdata_table.' custom_data ON custom_data.ResourceId = item_table.ResourceId
								LEFT JOIN '.TABLE_PREFIX.'CategoryItems ci ON ci.ItemResourceId = item_table.ResourceId
								LEFT JOIN '.TABLE_PREFIX.'Categories c ON c.CategoryId = ci.CategoryId
								WHERE '.$where_clause;
						$item_id = $this->Conn->GetOne($sql);
					}
					$save_method = $item_id && $this->curItem->Load($item_id) ? 'Update' : 'Create';
					if ($save_method == 'Update') {
						// replace id from csv file with found id (only when ID is found in cvs file)
						if (in_array($this->curItem->IDField, $this->exportFields)) {
							$record_data[ array_search($this->curItem->IDField, $this->exportFields) ] = $item_id;
						}
					}
				}

				$this->setImportData($record_data);
			}
			else {
				$this->resetImportObject($event, IMPORT_LIVE, $record_data);
				$category_id = $this->getItemCategory();
			}

			// create main record
			if ($save_method == 'Create') {
				$this->fillRequiredFields($this->false, $this->curItem, true);
			}

//			$sql_start = microtime(true);
			if (!$this->curItem->$save_method()) {
				$this->Application->SetVar('m_cat_id', $backup_category_id);
				return false;
			}
//			$sql_end = microtime(true);
//			$this->saveLog('SQL ['.$save_method.'] Time: '.($sql_end - $sql_start).'s');

			if ($load_keys && ($save_method == 'Create') && $this->exportOptions['ReplaceDuplicates']) {
				// map new id to old id
				$this->addToCache('new_ids', kUtil::crc32($where_clause), $this->curItem->GetID() );
			}

			// assign item to categories
			$this->curItem->assignToCategory($category_id, false);

			$this->Application->SetVar('m_cat_id', $backup_category_id);
			return true;
		}

		/*function saveLog($msg)
		{
			static $first_time = true;

			$fp = fopen((defined('RESTRICTED') ? RESTRICTED : FULL_PATH) . '/sqls.log', $first_time ? 'w' : 'a');
			fwrite($fp, $msg."\n");
			fclose($fp);

			$first_time = false;
		}*/

		/**
		 * Returns category parent path, if possible, then from cache
		 *
		 * @param int $category_id
		 * @return string
		 */
		function getParentPath($category_id)
		{
			$parent_path = $this->getFromCache('category_parent_path', $category_id);
			if ($parent_path === false) {
				$sql = 'SELECT ParentPath
						FROM '.TABLE_PREFIX.'Categories
						WHERE CategoryId = '.$category_id;
				$parent_path = $this->Conn->GetOne($sql);
				$this->addToCache('category_parent_path', $category_id, $parent_path);
			}
			return $parent_path;
		}

		function getFileExtension()
		{
			return $this->exportOptions['ExportFormat'] == 1 ? 'csv' : 'xml';
		}

		function getLineSeparator($option = 'LineEndings')
		{
			return $this->exportOptions[$option] == 1 ? "\r\n" : "\n";
		}

		/**
		 * Returns field caption for any exported field
		 *
		 * @param string $field
		 * @return string
		 */
		function getFieldCaption($field)
		{
			if (substr($field, 0, 10) == '__CUSTOM__')
			{
				$ret = 'Custom_'.substr($field, 10, strlen($field) );
			}
			elseif (substr($field, 0, 12) == '__CATEGORY__')
			{
				return $this->getCategoryTitle();
			}
			elseif (substr($field, 0, 11) == '__VIRTUAL__') {
				$ret = substr($field, 11);
			}
			else
			{
				$ret = $field;
			}

			return Array($ret);
		}

		/**
		 * Returns requested field value (including custom fields and category fields)
		 *
		 * @param string $field
		 * @return string
		 */
		function getFieldValue($field)
		{
			if (substr($field, 0, 10) == '__CUSTOM__') {
				$field = 'cust_'.substr($field, 10, strlen($field));
				$ret = $this->curItem->GetField($field);
			}
			elseif (substr($field, 0, 12) == '__CATEGORY__') {
				return $this->getCategoryPath();
			}
			elseif (substr($field, 0, 11) == '__VIRTUAL__') {
				$field = substr($field, 11);
				$ret = $this->curItem->GetField($field);
			}
			else
			{
				$ret = $this->curItem->GetField($field);
			}

			$ret = str_replace("\r\n", $this->getLineSeparator('LineEndingsInside'), $ret);
			return Array($ret);
		}

		/**
		 * Returns category field(-s) caption based on export mode
		 *
		 * @return string
		 */
		function getCategoryTitle()
		{
			// category path in separated fields
			$category_count = $this->getMaxCategoryLevel();
			if ($this->exportOptions['CategoryFormat'] == 1)
			{
				// category path in one field
				return $category_count ? Array('CategoryPath') : Array();
			}
			else
			{
				$i = 0;
				$ret = Array();
				while ($i < $category_count) {
					$ret[] = 'Category'.($i + 1);
					$i++;
				}
				return $ret;
			}
		}

		/**
		 * Returns category path in required format for current link
		 *
		 * @return string
		 */
		function getCategoryPath()
		{
			$category_id = $this->curItem->GetDBField('CategoryId');
			$category_path = $this->getFromCache('category_path', $category_id);

			if ( !$category_path ) {
				/** @var kMultiLanguage $ml_formatter */
				$ml_formatter = $this->Application->recallObject('kMultiLanguage');

				$sql = 'SELECT ' . $ml_formatter->LangFieldName('CachedNavbar') . '
						FROM ' . TABLE_PREFIX . 'Categories
						WHERE CategoryId = ' . $category_id;
				$category_path = $this->Conn->GetOne($sql);

				$category_path = $category_path ? explode('&|&', $category_path) : Array ();

				if ( $category_path && strtolower($category_path[0]) == 'content' ) {
					array_shift($category_path);
				}

				if ( $this->exportOptions['IsBaseCategory'] ) {
					$i = $this->exportOptions['BaseLevel'];
					while ( $i > 0 ) {
						array_shift($category_path);
						$i--;
					}
				}

				$category_count = $this->getMaxCategoryLevel();

				if ( $this->exportOptions['CategoryFormat'] == 1 ) {
					// category path in single field
					$category_path = $category_count ? Array (implode($this->exportOptions['CategorySeparator'], $category_path)) : Array ();
				}
				else {
					// category path in separated fields
					$levels_used = count($category_path);

					if ( $levels_used < $category_count ) {
						$i = 0;
						while ( $i < $category_count - $levels_used ) {
							$category_path[] = '';
							$i++;
						}
					}
				}
				$this->addToCache('category_path', $category_id, $category_path);
			}

			return $category_path;
		}

		/**
		 * Get maximal category deep level from links beeing exported
		 *
		 * @return int
		 */
		function getMaxCategoryLevel()
		{
			static $max_level = -1;

			if ($max_level != -1)
			{
				return $max_level;
			}

			$sql = 'SELECT IF(c.CategoryId IS NULL, 0, MAX( LENGTH(c.ParentPath) - LENGTH( REPLACE(c.ParentPath, "|", "") ) - 1 ))
					FROM '.$this->curItem->TableName.' item_table
					LEFT JOIN '.TABLE_PREFIX.'CategoryItems ci ON item_table.ResourceId = ci.ItemResourceId
					LEFT JOIN '.TABLE_PREFIX.'Categories c ON c.CategoryId = ci.CategoryId
					WHERE (ci.PrimaryCat = 1) AND ';

			$where_clause = '';
			if ($this->exportOptions['export_ids'] === false) {
				// get links from current category & all it's subcategories
				if ($this->exportOptions['export_cats_ids'][0] == 0) {
					$where_clause = 1;
				}
				else {
					foreach ($this->exportOptions['export_cats_ids'] as $category_id) {
						$where_clause .= '(c.ParentPath LIKE "%|'.$category_id.'|%") OR ';
					}
					$where_clause = substr($where_clause, 0, -4);
				}
			}
			else {
				// get only selected links
				$where_clause = $this->curItem->IDField.' IN ('.implode(',', $this->exportOptions['export_ids']).')';
			}

			$max_level = $this->Conn->GetOne($sql.'('.$where_clause.')');

			if ($this->exportOptions['IsBaseCategory'] ) {
				$max_level -= $this->exportOptions['BaseLevel'];
			}

			return $max_level;
		}

		/**
		 * Saves one record to export file
		 *
		 * @param Array $fields_hash
		 */
		function writeRecord($fields_hash)
		{
			kUtil::fputcsv($this->filePointer, $fields_hash, $this->exportOptions['FieldsSeparatedBy'], $this->exportOptions['FieldsEnclosedBy'], $this->getLineSeparator() );
		}

		function readRecord()
		{
			return fgetcsv($this->filePointer, 10000, $this->exportOptions['FieldsSeparatedBy'], $this->exportOptions['FieldsEnclosedBy']);
		}

		/**
		 * Saves import/export options
		 *
		 * @param kEvent $event
		 * @param Array $options
		 * @return void
		 */
		function saveOptions($event, $options = null)
		{
			if ( !isset($options) ) {
				$options = $this->exportOptions;
			}

			$this->Application->StoreVar($event->getPrefixSpecial() . '_options', serialize($options));
		}

		/**
		 * Loads import/export options
		 *
		 * @param kEvent $event
		 * @return Array
		 */
		function loadOptions($event)
		{
			return unserialize( $this->Application->RecallVar($event->getPrefixSpecial() . '_options') );
		}

		/**
		 * Sets correct available & export fields
		 *
		 * @param kEvent $event
		 */
		function prepareExportColumns($event)
		{
			/** @var kCatDBItem $object */
			$object = $event->getObject( Array('skip_autoload' => true) );

			if ( !$object->isField('ExportColumns') ) {
				// import/export prefix was used (see kDBEventHandler::prepareObject) but object don't plan to be imported/exported
				return ;
			}

			$available_columns = Array();

			if ($this->Application->getUnitOption($event->Prefix, 'CatalogItem')) {
				// category field (mixed)
				$available_columns['__CATEGORY__CategoryPath'] = 'CategoryPath';

				if ($event->Special == 'import') {
					// category field (separated fields)
					$max_level = $this->Application->ConfigValue('MaxImportCategoryLevels');
					$i = 0;
					while ($i < $max_level) {
						$available_columns['__CATEGORY__Category'.($i + 1)] = 'Category'.($i + 1);
						$i++;
					}
				}
			}

			// db fields
			$fields = $object->getFields();

			foreach ($fields as $field_name => $field_options) {
				if ( !$object->skipField($field_name) ) {
					$available_columns[$field_name] = $field_name.( $object->isRequired($field_name) ? '*' : '');
				}
			}

			/** @var kDBEventHandler $handler */
			$handler = $this->Application->recallObject($event->Prefix.'_EventHandler');

			$available_columns = array_merge($available_columns, $handler->getCustomExportColumns($event));

			// custom fields
			$custom_fields = $object->getCustomFields();

			foreach ($custom_fields as $custom_id => $custom_name)
			{
				$available_columns['__CUSTOM__'.$custom_name] = $custom_name;
			}

			// columns already in use
			$items_info = $this->Application->GetVar( $event->getPrefixSpecial(true) );
			if ($items_info)
			{
				$field_values = current($items_info);
				$export_keys = $field_values['ExportColumns'];
				$export_keys = $export_keys ? explode('|', substr($export_keys, 1, -1) ) : Array();
			}
			else {
				$export_keys = Array();
			}

			$export_columns = Array();
			foreach ($export_keys as $field_key)
			{
				$field_name = $this->getExportField($field_key);
				$export_columns[$field_key] = $field_name;
				unset($available_columns[$field_key]);
			}

			$options = $object->GetFieldOptions('ExportColumns');
			$options['options'] = $export_columns;
			$object->SetFieldOptions('ExportColumns', $options);

			$options = $object->GetFieldOptions('AvailableColumns');
			$options['options'] = $available_columns;
			$object->SetFieldOptions('AvailableColumns', $options);

			$this->updateImportFiles($event);
			$this->PrepareExportPresets($event);
		}

		/**
		 * Prepares export presets
		 *
		 * @param kEvent $event
		 * @return void
		 */
		function PrepareExportPresets($event)
		{
			/** @var kDBItem $object */
			$object = $event->getObject(Array ('skip_autoload' => true));

			$options = $object->GetFieldOptions('ExportPresets');
			$export_settings = $this->Application->RecallPersistentVar('export_settings');

			if ( !$export_settings ) {
				return;
			}

			$export_settings = unserialize($export_settings);

			if ( !isset($export_settings[$event->Prefix]) ) {
				return;
			}

			$export_presets = array ('' => '');

			foreach ($export_settings[$event->Prefix] as $key => $val) {
				$export_presets[implode('|', $val['ExportColumns'])] = $key;
			}

			$options['options'] = $export_presets;
			$object->SetFieldOptions('ExportPresets', $options);
		}

		function getExportField($field_key)
		{
			$prepends = Array('__CUSTOM__', '__CATEGORY__');
			foreach ($prepends as $prepend)
			{
				if (substr($field_key, 0, strlen($prepend) ) == $prepend)
				{
					$field_key = substr($field_key, strlen($prepend), strlen($field_key) );
					break;
				}
			}
			return $field_key;
		}

		/**
		 * Updates uploaded files list
		 *
		 * @param kEvent $event
		 * @return void
		 * @access protected
		 */
		protected function updateImportFiles($event)
		{
			if ( $event->Special != 'import' ) {
				return ;
			}

			/** @var FileHelper $file_helper */
			$file_helper = $this->Application->recallObject('FileHelper');

			$import_filenames = Array ();
			$file_helper->CheckFolder(EXPORT_PATH);

			$iterator = new DirectoryIterator(EXPORT_PATH);
			/** @var DirectoryIterator $file_info */

			foreach ($iterator as $file_info) {
				$file = $file_info->getFilename();

				if ( $file_info->isDir() || $file == 'dummy' || $file_info->getSize() == 0 ) {
					continue;
				}

				$import_filenames[$file] = $file . ' (' . kUtil::formatSize( $file_info->getSize() ) . ')';
			}

			/** @var kDBItem $object */
			$object = $event->getObject();

			$object->SetFieldOption('ImportLocalFilename', 'options', $import_filenames);
		}

		/**
		 * Returns module folder
		 *
		 * @param kEvent $event
		 * @return string
		 */
		function getModuleName($event)
		{
			$module_path = $this->Application->getUnitOption($event->Prefix, 'ModuleFolder') . '/';
			$module_name = $this->Application->findModule('Path', $module_path, 'Name');

			return mb_strtolower($module_name);
		}

		/**
		 * Export form validation & processing
		 *
		 * @param kEvent $event
		 */
		function OnExportBegin($event)
		{
			$items_info = $this->Application->GetVar($event->getPrefixSpecial(true));

			if ( !$items_info ) {
				$items_info = unserialize($this->Application->RecallVar($event->getPrefixSpecial() . '_ItemsInfo'));
				$this->Application->SetVar($event->getPrefixSpecial(true), $items_info);
			}

			$item_id = key($items_info);
			$field_values = $items_info[$item_id];

			/** @var kDBItem $object */
			$object = $event->getObject(Array ('skip_autoload' => true));

			$object->SetFieldsFromHash($field_values);
			$field_values['ImportFilename'] = $object->GetDBField('ImportFilename'); //if upload formatter has renamed the file during moving !!!

			$object->setID($item_id);
			$this->setRequiredFields($event);

			// save export/import options
			if ( $event->Special == 'export' ) {
				$export_ids = $this->Application->RecallVar($event->Prefix . '_export_ids');
				$export_cats_ids = $this->Application->RecallVar($event->Prefix . '_export_cats_ids');

				// used for multistep export
				$field_values['export_ids'] = $export_ids ? explode(',', $export_ids) : false;
				$field_values['export_cats_ids'] = $export_cats_ids ? explode(',', $export_cats_ids) : Array ($this->Application->GetVar('m_cat_id'));
				$field_values['export_special'] = $this->Application->RecallVar('export_special');
				$field_values['export_grid'] = $this->Application->RecallVar('export_grid');
			}

			$field_values['ExportColumns'] = $field_values['ExportColumns'] ? explode('|', substr($field_values['ExportColumns'], 1, -1) ) : Array();
			$field_values['start_from'] = 0;

			$nevent = new kEvent($event->Prefix . ':OnBeforeExportBegin');
			$nevent->setEventParam('options', $field_values);
			$this->Application->HandleEvent($nevent);
			$field_values = $nevent->getEventParam('options');

			$this->saveOptions($event, $field_values);

			if ( $this->verifyOptions($event) ) {
				if ( $this->_getExportSavePreset($object) ) {
					$name = $object->GetDBField('ExportPresetName');

					$export_settings = $this->Application->RecallPersistentVar('export_settings');
					$export_settings = $export_settings ? unserialize($export_settings) : array ();
					$export_settings[$event->Prefix][$name] = $field_values;
					$this->Application->StorePersistentVar('export_settings', serialize($export_settings));
				}

				$progress_t = $this->Application->RecallVar('export_progress_t');
				if ( $progress_t ) {
					$this->Application->RemoveVar('export_progress_t');
				}
				else {
					$progress_t = $this->getModuleName($event) . '/' . $event->Special . '_progress';
				}
				$event->redirect = $progress_t;

				if ( $event->Special == 'import' ) {
					$import_category = (int)$this->Application->RecallVar('ImportCategory');

					// in future could use module root category if import category will be unavailable :)
					$event->SetRedirectParam('m_cat_id', $import_category); // for template permission checking
					$this->Application->StoreVar('m_cat_id', $import_category); // for event permission checking
				}
			}
			else {
				// make uploaded file local & change source selection
				$filename = getArrayValue($field_values, 'ImportFilename');

				if ( $filename ) {
					$this->updateImportFiles($event);
					$object->SetDBField('ImportSource', 2);
					$field_values['ImportSource'] = 2;
					$object->SetDBField('ImportLocalFilename', $filename);
					$field_values['ImportLocalFilename'] = $filename;
					$this->saveOptions($event, $field_values);
				}

				$event->status = kEvent::erFAIL;
				$event->redirect = false;
			}
		}

		/**
		 * Returns export save preset name, when used at all
		 *
		 * @param kDBItem $object
		 * @return string
		 */
		function _getExportSavePreset(&$object)
		{
			if ( !$object->isField('ExportSavePreset') ) {
				return '';
			}

			return $object->GetDBField('ExportSavePreset');
		}

		/**
		 * set required fields based on import or export params
		 *
		 * @param kEvent $event
		 */
		function setRequiredFields($event)
		{
			$required_fields['common'] = Array('FieldsSeparatedBy', 'LineEndings', 'CategoryFormat');

			$required_fields['export'] = Array('ExportFormat', 'ExportFilename','ExportColumns');

			/** @var kDBItem $object */
			$object = $event->getObject();

			if ($this->_getExportSavePreset($object)) {
				$required_fields['export'][] = 'ExportPresetName';
			}

			$required_fields['import'] = Array('FieldTitles', 'ImportSource', 'CheckDuplicatesMethod'); // ImportFilename, ImportLocalFilename

			if ($event->Special == 'import')
			{
				$import_source = Array(1 => 'ImportFilename', 2 => 'ImportLocalFilename');
				$used_field = $import_source[ $object->GetDBField('ImportSource') ];

				$required_fields[$event->Special][] = $used_field;
				$object->SetFieldOption($used_field, 'error_field', 'ImportSource');

				if ($object->GetDBField('FieldTitles') == 2) $required_fields[$event->Special][] = 'ExportColumns'; // manual field titles
			}

			$required_fields = array_merge($required_fields['common'], $required_fields[$event->Special]);
			$object->setRequired($required_fields);
		}

	}