<?php
/**
* @version	$Id: csv_helper.php 16513 2017-01-20 14:10:53Z alex $
* @package	In-Portal
* @copyright	Copyright (C) 1997 - 2009 Intechnic. All rights reserved.
* @license      GNU/GPL
* In-Portal is Open Source software.
* This means that this software may have been modified pursuant
* the GNU General Public License, and as distributed it includes
* or is derivative of works licensed under the GNU General Public License
* or other free or open source software licenses.
* See http://www.in-portal.org/license for copyright notices and details.
*/

	defined('FULL_PATH') or die('restricted access!');

	kUtil::safeDefine('EXPORT_STEP', 100); // export by 100 items
	kUtil::safeDefine('IMPORT_STEP', 10);

	class kCSVHelper extends kHelper {

		var $PrefixSpecial;
		var $grid;

		var $delimiter_mapping = Array(0 => "\t", 1 => ',', 2 => ';', 3 => ' ', 4 => ':');
		var $enclosure_mapping = Array(0 => '"', 1 => "'");
		var $separator_mapping = Array(0 => "\n", 1 => "\r\n");

		function ExportStep()
		{
			$export_data = $this->Application->RecallVar('export_data');
			$export_rand = $this->Application->RecallVar('export_rand');
			$get_rand = $this->Application->GetVar('export_rand');

			/** @var FileHelper $file_helper */
			$file_helper = $this->Application->recallObject('FileHelper');

			if ( $export_data && $export_rand == $get_rand ) {
				$export_data = unserialize($export_data);
				$first_step = false;
			}
			else {
				// first step
				$export_data = Array ();
				$export_data['prefix'] = $this->PrefixSpecial;
				$export_data['grid'] = $this->grid;
				$export_data['file_name'] = EXPORT_PATH . '/' . $file_helper->ensureUniqueFilename(EXPORT_PATH, 'export_' . $export_data['prefix'] . '.csv');
				$export_data['step'] = EXPORT_STEP;
				$export_data['delimiter'] = $this->delimiter_mapping[(int)$this->Application->ConfigValue('CSVExportDelimiter')];
				$export_data['enclosure'] = $this->enclosure_mapping[(int)$this->Application->ConfigValue('CSVExportEnclosure')];
				$export_data['record_separator'] = $this->separator_mapping[(int)$this->Application->ConfigValue('CSVExportSeparator')];
				$export_data['page'] = 1;
				$export_data['source_encoding'] = strtoupper(CHARSET);
				$export_data['encoding'] = $this->Application->ConfigValue('CSVExportEncoding') ? false : 'UTF-16LE';

				$this->Application->StoreVar('export_rand', $get_rand);

				$first_step = true;
			}

			$file = fopen($export_data['file_name'], $first_step ? 'w' : 'a');

			$prefix_elems = preg_split('/\.|_/', $export_data['prefix'], 2);
			$grids = $this->Application->getUnitOption($prefix_elems[0], 'Grids');
			$grid_config = $grids[$export_data['grid']]['Fields'];

			$list_params = Array ('per_page' => $export_data['step'], 'grid' => $export_data['grid']);

			/** @var kDBList $list */
			$list = $this->Application->recallObject(rtrim(implode('.', $prefix_elems), '.'), $prefix_elems[0] . '_List', $list_params);

			$list->SetPage($export_data['page']);
			$list->Query();
			$list->GoFirst();

			$picker_helper = new kColumnPickerHelper(rtrim(implode('.', $prefix_elems), '.'), $export_data['grid']);
			$grid_config = $picker_helper->apply($grid_config);

			if ( $first_step ) {
				// if UTF-16, write Unicode marker
				if ( $export_data['encoding'] == 'UTF-16LE' ) {
					fwrite($file, chr(0xFF) . chr(0xFE));
				}

				// inserting header line
				$headers = Array ();
				foreach ($grid_config as $field_name => $field_data) {
					$use_phrases = array_key_exists('use_phrases', $field_data) ? $field_data['use_phrases'] : true;
					$field_title = isset($field_data['title']) ? $field_data['title'] : 'column:la_fld_' . $field_name;
					$header = $use_phrases ? $this->Application->Phrase($field_title) : $field_title;
					array_push($headers, $header);
				}

				$csv_line = kUtil::getcsvline($headers, $export_data['delimiter'], $export_data['enclosure'], $export_data['record_separator']);
				if ( $export_data['encoding'] ) {
					$csv_line = mb_convert_encoding($csv_line, $export_data['encoding'], $export_data['source_encoding']);
				}
				fwrite($file, $csv_line);
			}

			while (!$list->EOL()) {
				$data = Array ();
				foreach ($grid_config as $field_name => $field_data) {
					if ( isset($field_data['export_field']) ) {
						$field_name = $field_data['export_field'];
					}
					$value = $list->GetField($field_name, isset($field_data['format']) ? $field_data['format'] : null);
					$value = str_replace("\r\n", "\n", $value);
					$value = str_replace("\r", "\n", $value);
					array_push($data, $value);
				}

				if ( $export_data['encoding'] == 'UTF-16LE' ) {
					fwrite($file, chr(0xFF) . chr(0xFE));
				}

				$csv_line = kUtil::getcsvline($data, $export_data['delimiter'], $export_data['enclosure'], $export_data['record_separator']);
				if ( $export_data['encoding'] ) {
					$csv_line = mb_convert_encoding($csv_line, $export_data['encoding'], $export_data['source_encoding']);
				}
				fwrite($file, $csv_line);

				$list->GoNext();
			}

			$records_processed = $export_data['page'] * $export_data['step'];
			$percent_complete = min($records_processed / $list->GetRecordsCount() * 100, 100);

			fclose($file);

			if ( $records_processed >= $list->GetRecordsCount() ) {
				$this->Application->StoreVar('export_data', serialize($export_data));
				$this->Application->Redirect($this->Application->GetVar('finish_template'));
			}

			echo $percent_complete;

			$export_data['page']++;
			$this->Application->StoreVar('export_data', serialize($export_data));
		}

		function ExportData($name)
		{
			$export_data = unserialize($this->Application->RecallVar('export_data'));
			return isset($export_data[$name]) ? $export_data[$name] : false;
		}

		/**
		 * Returns prefix from request or from stored import/export data
		 *
		 * @param bool $is_import
		 * @return string
		 */
		public function getPrefix($is_import = false)
		{
			$prefix = $this->Application->GetVar('PrefixSpecial');

			if ( !$prefix ) {
				return $is_import ? $this->ImportData('prefix') : $this->ExportData('prefix');
			}

			return $prefix;
		}

		function GetCSV()
		{
			kUtil::safeDefine('DBG_SKIP_REPORTING', 1);

			$export_data = unserialize($this->Application->RecallVar('export_data'));
			$filename = preg_replace('/(.*)\.csv$/', '\1', basename($export_data['file_name'])) . '.csv';

			$this->Application->setContentType('text/csv');
			header('Content-Disposition: attachment; filename="' . $filename . '"');
			readfile($export_data['file_name']);
			die();
		}

		function ImportStart($filename)
		{
			if(!file_exists($filename) || !is_file($filename)) return 'cant_open_file';

			$import_data = Array();
			$import_data['source_encoding'] = strtoupper(CHARSET);
			$import_data['encoding'] = $this->Application->ConfigValue('CSVExportEncoding') ? false : 'UTF-16LE';
			$import_data['errors'] = '';

			// convert file in case of UTF-16LE
			if($import_data['source_encoding'] != $import_data['encoding']) {
				copy($filename, $filename.'.orginal');
				$file_content = file_get_contents($filename);
				$file = fopen($filename, 'w');
				fwrite($file, mb_convert_encoding(str_replace(chr(0xFF).chr(0xFE), '', $file_content), $import_data['source_encoding'], $import_data['encoding']));
				fclose($file);

			}

			$import_data['prefix'] = $this->PrefixSpecial;
			$import_data['grid'] = $this->grid;
			$import_data['file'] = $filename;
			$import_data['total_lines'] = count(file($filename));
			if(!$import_data['total_lines']) $import_data['total_lines'] = 1;
			unset($file_content);
			$import_data['lines_processed'] = 0;
			$import_data['delimiter'] = $this->delimiter_mapping[(int)$this->Application->ConfigValue('CSVExportDelimiter')];
			$import_data['enclosure'] = $this->enclosure_mapping[(int)$this->Application->ConfigValue('CSVExportEnclosure')];
			$import_data['step'] = IMPORT_STEP;
			$import_data['not_imported_lines'] = '';
			$import_data['added'] = 0;
			$import_data['updated'] = 0;

			$file = fopen($filename, 'r');
			// getting first line for headers
			$headers = fgetcsv($file, 8192, $import_data['delimiter'], $import_data['enclosure']);
			fclose($file);

			$prefix_elems = preg_split('/\.|_/', $import_data['prefix'], 2);
			$grids = $this->Application->getUnitOption($prefix_elems[0], 'Grids');
			$grid_config = $grids[ $import_data['grid'] ]['Fields'];

			$field_list = Array();
			foreach($grid_config as $field_name => $field_data) {
				if(isset($field_data['export_field'])) {
					$field_name = $field_data['export_field'];
				}
				$field_title = isset($field_data['title']) ? $field_data['title'] : 'column:la_fld_' . $field_name;
				$field_label = $this->Application->Phrase($field_title);
				$field_pos = array_search($field_label, $headers);
				if($field_pos !== false) {
					$field_list[$field_pos] = $field_name;
				}
			}

			if(!count($field_list)) return 'no_matching_columns';
			$import_data['field_list'] = $field_list;

			// getting key list
			$field_positions = Array();
			$config_key_list = $this->Application->getUnitOption($prefix_elems[0], 'ImportKeys');
			if(!$config_key_list) $config_key_list = Array();
			array_unshift($config_key_list, Array($this->Application->getUnitOption($prefix_elems[0], 'IDField')));

			$key_list = Array();
			foreach($config_key_list as $arr_key => $import_key) {
				$key_list[$arr_key] = is_array($import_key) ? $import_key : Array($import_key);

				foreach($key_list[$arr_key] as $key_field) {
					$field_positions[$key_field] = array_search($key_field, $import_data['field_list']);
					if($field_positions[$key_field] === false) {
						// no such key field combination in imported file
						unset($key_list[$arr_key]);
						break;
					}
				}
			}
			$import_data['key_list'] = $key_list;
			$import_data['field_positions'] = $field_positions;

			$this->Application->StoreVar('import_data', serialize($import_data));
			return true;
		}

		function ImportStep()
		{
			$import_data = unserialize($this->Application->RecallVar('import_data'));
			$prefix_elems = preg_split('/\.|_/', $import_data['prefix'], 2);

			/** @var kDBItem $object */
			$object = $this->Application->recallObject($prefix_elems[0].'.-csvimport', $prefix_elems[0], Array('skip_autoload' => true, 'populate_ml_fields' => true));

			$file = fopen($import_data['file'], 'r');
			$eof = false;
			// skipping lines that has been already imported
			for($i = 0; $i < $import_data['lines_processed'] + 1; $i++) {
				if(feof($file)) break;
				fgets($file, 8192);
			}

			$import_event = new kEvent($prefix_elems[0].'.-csvimport:OnBeforeCSVLineImport');

			for($i = 0; $i < $import_data['step']; $i++) {
				if(feof($file)) break;
				$data = fgetcsv($file, 8192, $import_data['delimiter'], $import_data['enclosure']);
				if(!$data) continue;

				$object->Clear();
				$action = 'Create';

				// 1. trying to load object by keys
				foreach($import_data['key_list'] as $key) {
					$fail = false;
					$key_array = Array();
					foreach($key as $key_field) {
						if(!isset($data[ $import_data['field_positions'][$key_field] ])) {
							$fail = true;
							break;
						}
						$key_array[$key_field] = $data[ $import_data['field_positions'][$key_field] ];
					}
					if($fail) continue;
					if($object->Load($key_array)) {
						$action = 'Update';
						break;
					}
				}

				// 2. set object fields
				foreach($import_data['field_list'] as $position => $field_name) {
					if(isset($data[$position])) {
						$object->SetField($field_name, $data[$position]);
					}
				}

				// 3. validate item and run event
				$status = $object->Validate();
				$import_event->status = $status ? kEvent::erSUCCESS : kEvent::erFAIL;
				$this->Application->HandleEvent($import_event);

				if($import_event->status == kEvent::erSUCCESS && $object->$action()) {
					$import_data[ ($action == 'Create') ? 'added' : 'updated' ]++;
				}
				else {
					$msg = '';
					$errors = $object->GetFieldErrors();

					foreach ($errors as $field => $info) {
						if (!$info['pseudo']) continue;
						$msg .= "$field: {$info['pseudo']} ";
					}

					$import_data['errors'] .= ($i + $import_data['lines_processed'] + 1).": $msg\n";
					$import_data['not_imported_lines'] .= ','.($i + $import_data['lines_processed'] + 1);
				}
			}

			$import_data['lines_processed'] += $import_data['step'];

			$import_data['not_imported_lines'] = ltrim($import_data['not_imported_lines'], ',');
			$this->Application->StoreVar('import_data', serialize($import_data));

			$feof = feof($file);
			fclose($file);

			if($feof) {
				$this->Application->Redirect($this->Application->GetVar('finish_template'));
			}
			else {
				$percent_complete = floor($import_data['lines_processed'] / $import_data['total_lines'] * 100);
				if($percent_complete > 99) $percent_complete = 99;
				echo $percent_complete;
			}
		}

		function ImportData($name)
		{
			$import_data = unserialize($this->Application->RecallVar('import_data'));
			return isset($import_data[$name]) ? $import_data[$name] : false;
		}

		function GetNotImportedLines()
		{
			$import_data = unserialize($this->Application->RecallVar('import_data'));

			if(!$import_data['not_imported_lines']) return false;
			$line_numbers = explode(',', $import_data['not_imported_lines']);
			$line_numbers[] = 0; // include header row in output

			$file = fopen($import_data['file'], 'r');
			$eof = false;
			$result = '';
			for($i = 0; $i <= max($line_numbers); $i++) {
				if(feof($file)) break;
				$line = fgets($file, 8192);
				if(in_array($i, $line_numbers)) {
					$result .= $i.':'.$line;
				}
			}
			return $result."\n\n".$import_data['errors'];
		}
	}