<?php
/**
* @version	$Id: csv_helper.php 13569 2010-05-18 15:25:01Z alex $
* @package	In-Portal
* @copyright	Copyright (C) 1997 - 2009 Intechnic. All rights reserved.
* @license      GNU/GPL
* In-Portal is Open Source software.
* This means that this software may have been modified pursuant
* the GNU General Public License, and as distributed it includes
* or is derivative of works licensed under the GNU General Public License
* or other free or open source software licenses.
* See http://www.in-portal.org/license for copyright notices and details.
*/

	defined('FULL_PATH') or die('restricted access!');

	safeDefine('EXPORT_STEP', 100); // export by 100 items
	safeDefine('IMPORT_STEP', 10);

	class kCSVHelper extends kHelper {

		var $PrefixSpecial;
		var $grid;

		var $delimiter_mapping = Array(0 => "\t", 1 => ',', 2 => ';', 3 => ' ', 4 => ':');
		var $enclosure_mapping = Array(0 => '"', 1 => "'");
		var $separator_mapping = Array(0 => "\n", 1 => "\r\n");

		function ExportStep()
		{
			$export_data = $this->Application->RecallVar('export_data');
			$export_rand = $this->Application->RecallVar('export_rand');
			$get_rand = $this->Application->GetVar('export_rand');

			if($export_data && $export_rand == $get_rand) {
				$export_data = unserialize($export_data);
				$first_step = false;
			}
			else {
				// first step
				$export_data = Array();
				$export_data['prefix'] = $this->PrefixSpecial;
				$export_data['grid'] = $this->grid;
				$export_data['file_name'] = EXPORT_PATH.'/'.$this->ValidateFileName(EXPORT_PATH, 'export_'.$export_data['prefix'].'.csv');
				$export_data['step'] = EXPORT_STEP;
				$export_data['delimiter'] = $this->delimiter_mapping[(int)$this->Application->ConfigValue('CSVExportDelimiter')];
				$export_data['enclosure'] = $this->enclosure_mapping[(int)$this->Application->ConfigValue('CSVExportEnclosure')];
				$export_data['record_separator'] = $this->separator_mapping[(int)$this->Application->ConfigValue('CSVExportSeparator')];
				$export_data['page'] = 1;

				$lang_object =& $this->Application->recallObject('lang.current');
				/* @var $lang_object LanguagesItem */
				$export_data['source_encoding'] = strtoupper( $lang_object->GetDBField('Charset') );
				$export_data['encoding'] = $this->Application->ConfigValue('CSVExportEncoding') ? false : 'UTF-16LE';

				$this->Application->StoreVar('export_rand', $get_rand);

				$first_step = true;
			}

			$file_helper =& $this->Application->recallObject('FileHelper');
			/* @var $file_helper FileHelper */

			$file_helper->CheckFolder( dirname($export_data['file_name']) );

			$file = fopen($export_data['file_name'], $first_step ? 'w' : 'a');

			$prefix_elems = split('\.|_', $export_data['prefix']);
			$grids = $this->Application->getUnitOption($prefix_elems[0], 'Grids');
			$grid_config = $grids[ $export_data['grid'] ]['Fields'];

			$list_params = Array('per_page' => $export_data['step'], 'grid' => $export_data['grid']);
			$list =& $this->Application->recallObject(rtrim(implode('.', $prefix_elems), '.'), $prefix_elems[0].'_List', $list_params);
			/* @var $list kDBList */
			$list->SetPage($export_data['page']);

			$list->Query();
			$list->GoFirst();

			$picker_helper =& $this->Application->RecallObject('ColumnPickerHelper');
			/* @var $picker_helper kColumnPickerHelper */
			$picker_helper->ApplyPicker(rtrim(implode('.', $prefix_elems), '.'), $grid_config, $export_data['grid']);

			if($first_step) {
				// if UTF-16, write Unicode marker
				if($export_data['encoding'] == 'UTF-16LE') {
					fwrite($file, chr(0xFF).chr(0xFE));
				}

				// inserting header line
				$headers = Array();
				foreach($grid_config as $field_name => $field_data) {
					$header = $this->Application->Phrase( $field_data['title'] );
					array_push($headers, $header);
				}
				$csv_line = getcsvline($headers, $export_data['delimiter'], $export_data['enclosure'], $export_data['record_separator']);
				if($export_data['encoding']) {
					$csv_line = mb_convert_encoding($csv_line, $export_data['encoding'], $export_data['source_encoding']);
				}
				fwrite($file, $csv_line);
			}

			while(!$list->EOL())
			{
				$data = Array();
				foreach($grid_config as $field_name => $field_data) {
					if(isset($field_data['export_field'])) {
						$field_name = $field_data['export_field'];
					}
					$value = $list->GetField($field_name, isset($field_data['format']) ? $field_data['format'] : null);
					$value = str_replace("\r\n", "\n", $value);
					$value = str_replace("\r", "\n", $value);
					array_push($data, $value);
				}
				if($export_data['encoding'] == 'UTF-16LE') {
					fwrite($file, chr(0xFF).chr(0xFE));
				}
				$csv_line = getcsvline($data, $export_data['delimiter'], $export_data['enclosure'], $export_data['record_separator']);
				if($export_data['encoding']) {
					$csv_line = mb_convert_encoding($csv_line, $export_data['encoding'], $export_data['source_encoding']);
				}
				fwrite($file, $csv_line);

				$list->GoNext();
			}

			$records_processed = $export_data['page'] * $export_data['step'];
			$percent_complete = min($records_processed / $list->RecordsCount * 100, 100);

			fclose($file);

			if ($records_processed >= $list->RecordsCount) {
				$this->Application->StoreVar('export_data', serialize($export_data));
				$this->Application->Redirect($this->Application->GetVar('finish_template'));
			}

			echo $percent_complete;

			$export_data['page']++;
			$this->Application->StoreVar('export_data', serialize($export_data));
		}

		function ValidateFileName($path, $name)
		{
			$parts = pathinfo($name);
			$ext = '.'.$parts['extension'];
			$filename = mb_substr($parts['basename'], 0, -mb_strlen($ext));
			$new_name = $filename.$ext;
			while ( file_exists($path.'/'.$new_name) )
			{
		   		if ( preg_match('/('.preg_quote($filename, '/').'_)([0-9]*)('.preg_quote($ext, '/').')/', $new_name, $regs) ) {
						$new_name = $regs[1].($regs[2]+1).$regs[3];
					}
					else {
						$new_name = $filename.'_1'.$ext;
					}
			}
			return $new_name;
		}

		function ExportData($name)
		{
			$export_data = unserialize($this->Application->RecallVar('export_data'));
			return isset($export_data[$name]) ? $export_data[$name] : false;
		}

		function GetCSV()
		{
			safeDefine('DBG_SKIP_REPORTING', 1);

			$export_data = unserialize($this->Application->RecallVar('export_data'));
			$filename = preg_replace('/(.*)\.csv$/', '\1', basename($export_data['file_name'])) . '.csv';

			header('Content-type: text/csv');
			header('Content-Disposition: attachment; filename="' . $filename . '"');
			readfile($export_data['file_name']);
			die();
		}

		function ImportStart($filename)
		{
			if(!file_exists($filename) || !is_file($filename)) return 'cant_open_file';

			$import_data = Array();

			$lang_object =& $this->Application->recallObject('lang.current');
			/* @var $lang_object LanguagesItem */
			$import_data['source_encoding'] = strtoupper( $lang_object->GetDBField('Charset') );
			$import_data['encoding'] = $this->Application->ConfigValue('CSVExportEncoding') ? false : 'UTF-16LE';
			$import_data['errors'] = '';

			// convert file in case of UTF-16LE
			if($import_data['source_encoding'] != $import_data['encoding']) {
				copy($filename, $filename.'.orginal');
				$file_content = file_get_contents($filename);
				$file = fopen($filename, 'w');
				fwrite($file, mb_convert_encoding(str_replace(chr(0xFF).chr(0xFE), '', $file_content), $import_data['source_encoding'], $import_data['encoding']));
				fclose($file);

			}

			$import_data['prefix'] = $this->PrefixSpecial;
			$import_data['grid'] = $this->grid;
			$import_data['file'] = $filename;
			$import_data['total_lines'] = count(file($filename));
			if(!$import_data['total_lines']) $import_data['total_lines'] = 1;
			unset($file_content);
			$import_data['lines_processed'] = 0;
			$import_data['delimiter'] = $this->delimiter_mapping[(int)$this->Application->ConfigValue('CSVExportDelimiter')];
			$import_data['enclosure'] = $this->enclosure_mapping[(int)$this->Application->ConfigValue('CSVExportEnclosure')];
			$import_data['step'] = IMPORT_STEP;
			$import_data['not_imported_lines'] = '';
			$import_data['added'] = 0;
			$import_data['updated'] = 0;

			$file = fopen($filename, 'r');
			// getting first line for headers
			$headers = fgetcsv($file, 8192, $import_data['delimiter'], $import_data['enclosure']);
			fclose($file);

			$prefix_elems = split('\.|_', $import_data['prefix']);
			$grids = $this->Application->getUnitOption($prefix_elems[0], 'Grids');
			$grid_config = $grids[ $import_data['grid'] ]['Fields'];

			$field_list = Array();
			foreach($grid_config as $field_name => $field_data) {
				if(isset($field_data['export_field'])) {
					$field_name = $field_data['export_field'];
				}
				$field_label = $this->Application->Phrase( $field_data['title'] );
				$field_pos = array_search($field_label, $headers);
				if($field_pos !== false) {
					$field_list[$field_pos] = $field_name;
				}
			}

			if(!count($field_list)) return 'no_matching_columns';
			$import_data['field_list'] = $field_list;

			// getting key list
			$field_positions = Array();
			$config_key_list = $this->Application->getUnitOption($prefix_elems[0], 'ImportKeys');
			if(!$config_key_list) $config_key_list = Array();
			array_unshift($config_key_list, Array($this->Application->getUnitOption($prefix_elems[0], 'IDField')));

			$key_list = Array();
			foreach($config_key_list as $arr_key => $import_key) {
				$key_list[$arr_key] = is_array($import_key) ? $import_key : Array($import_key);

				foreach($key_list[$arr_key] as $key_field) {
					$field_positions[$key_field] = array_search($key_field, $import_data['field_list']);
					if($field_positions[$key_field] === false) {
						// no such key field combination in imported file
						unset($key_list[$arr_key]);
						break;
					}
				}
			}
			$import_data['key_list'] = $key_list;
			$import_data['field_positions'] = $field_positions;

			$this->Application->StoreVar('import_data', serialize($import_data));
			return true;
		}

		function ImportStep()
		{
			$import_data = unserialize($this->Application->RecallVar('import_data'));
			$prefix_elems = split('\.|_', $import_data['prefix']);

			$object =& $this->Application->recallObject($prefix_elems[0].'.-csvimport', $prefix_elems[0], Array('skip_autoload' => true, 'populate_ml_fields' => true));
			/* @var $object kDBItem */

			$file = fopen($import_data['file'], 'r');
			$eof = false;
			// skipping lines that has been already imported
			for($i = 0; $i < $import_data['lines_processed'] + 1; $i++) {
				if(feof($file)) break;
				fgets($file, 8192);
			}

			$import_event = new kEvent($prefix_elems[0].'.-csvimport:OnBeforeCSVLineImport');

			for($i = 0; $i < $import_data['step']; $i++) {
				if(feof($file)) break;
				$data = fgetcsv($file, 8192, $import_data['delimiter'], $import_data['enclosure']);
				if(!$data) continue;

				$object->Clear();
				$action = 'Create';

				// 1. trying to load object by keys
				foreach($import_data['key_list'] as $key) {
					$fail = false;
					$key_array = Array();
					foreach($key as $key_field) {
						if(!isset($data[ $import_data['field_positions'][$key_field] ])) {
							$fail = true;
							break;
						}
						$key_array[$key_field] = $data[ $import_data['field_positions'][$key_field] ];
					}
					if($fail) continue;
					if($object->Load($key_array)) {
						$action = 'Update';
						break;
					}
				}

				// 2. set object fields
				foreach($import_data['field_list'] as $position => $field_name) {
					if(isset($data[$position])) {
						$object->SetField($field_name, $data[$position]);
					}
				}

				// 3. validate item and run event
				$status = $object->Validate();
				$import_event->status = $status ? erSUCCESS : erFAIL;
				$this->Application->HandleEvent($import_event);

				if($import_event->status == erSUCCESS && $object->$action()) {
					$import_data[ ($action == 'Create') ? 'added' : 'updated' ]++;
				}
				else {
					$msg = '';
					foreach ($object->FieldErrors as $field => $info) {
						if (!$info['pseudo']) continue;
						$msg .= "$field: {$info['pseudo']} ";
					}
					$import_data['errors'] .= ($i + $import_data['lines_processed'] + 1).": $msg\n";
					$import_data['not_imported_lines'] .= ','.($i + $import_data['lines_processed'] + 1);
				}
			}

			$import_data['lines_processed'] += $import_data['step'];

			$import_data['not_imported_lines'] = ltrim($import_data['not_imported_lines'], ',');
			$this->Application->StoreVar('import_data', serialize($import_data));

			$feof = feof($file);
			fclose($file);

			if($feof) {
				$this->Application->Redirect($this->Application->GetVar('finish_template'));
			}
			else {
				$percent_complete = floor($import_data['lines_processed'] / $import_data['total_lines'] * 100);
				if($percent_complete > 99) $percent_complete = 99;
				echo $percent_complete;
			}
		}

		function ImportData($name)
		{
			$import_data = unserialize($this->Application->RecallVar('import_data'));
			return isset($import_data[$name]) ? $import_data[$name] : false;
		}

		function GetNotImportedLines()
		{
			$import_data = unserialize($this->Application->RecallVar('import_data'));

			if(!$import_data['not_imported_lines']) return false;
			$line_numbers = explode(',', $import_data['not_imported_lines']);
			$line_numbers[] = 0; // include header row in output

			$file = fopen($import_data['file'], 'r');
			$eof = false;
			$result = '';
			for($i = 0; $i <= max($line_numbers); $i++) {
				if(feof($file)) break;
				$line = fgets($file, 8192);
				if(in_array($i, $line_numbers)) {
					$result .= $i.':'.$line;
				}
			}
			return $result."\n\n".$import_data['errors'];
		}
	}