Index: trunk/core/units/general/cat_dbitem_export.php =================================================================== diff -u -r3765 -r3787 --- trunk/core/units/general/cat_dbitem_export.php (.../cat_dbitem_export.php) (revision 3765) +++ trunk/core/units/general/cat_dbitem_export.php (.../cat_dbitem_export.php) (revision 3787) @@ -2,60 +2,60 @@ define('EXPORT_STEP', 200); // export by 200 items (e.g. links) define('IMPORT_CHUNK', 50120); // 5 KB - + define('IMPORT_TEMP', 1); define('IMPORT_LIVE', 2); - + class kCatDBItemExportHelper extends kHelper { - + var $false = false; - + var $cache = Array(); - + var $exportFields = Array(); - + /** * Export options * * @var Array */ var $exportOptions = Array(); - + /** * If we have custom fields in export * * @var unknown_type */ var $hasCustomFields = false; - + /** * Custom field values for last item beeing exported * * @var Array */ var $customValues = Array(); - + /** * Item beeing currenly exported * * @var kCatDBItem */ var $curItem = null; - + /** * Dummy category object * * @var CategoriesItem */ var $dummyCategory = null; - + /** * Pointer to opened file * * @var resource */ var $filePointer = null; - + /** * Returns value from cache if found or false otherwise * @@ -67,7 +67,7 @@ { return getArrayValue($this->cache, $type, $key); } - + /** * Adds value to be cached * @@ -80,7 +80,7 @@ // if (!isset($this->cache[$type])) $this->cache[$type] = Array(); $this->cache[$type][$key] = $value; } - + /** * Fill required fields with dummy values * @@ -91,31 +91,31 @@ if ($object == $this->false) { $object =& $event->getObject(); } - + $has_empty = false; $fields = array_keys($object->Fields); foreach ($fields as $field_name) { $field_options =& $object->Fields[$field_name]; if (isset($object->VirtualFields[$field_name]) || !getArrayValue($field_options, 'required') ) continue; if ( $object->GetDBField($field_name) ) continue; - + $formatter_class = getArrayValue($field_options, 'formatter'); if ($formatter_class) // not tested { $formatter =& $this->Application->recallObject($formatter_class); $sample_value = $formatter->GetSample($field_name, $field_options, $object); } - + $has_empty = true; $object->SetDBField($field_name, isset($sample_value) && $sample_value ? $sample_value : 'no value'); } - + if ($set_status && $has_empty) { $object->SetDBField('Status', 0); } } - + /** * Verifies that all user entered export params are correct * @@ -128,27 +128,27 @@ $this->Application->StoreVar($event->getPrefixSpecial().'_ForceNotValid', 0); return false; } - + $this->fillRequiredFields($event, $this->false); - + $object =& $event->getObject(); $cross_unique_fields = Array('FieldsSeparatedBy', 'FieldsEnclosedBy'); if (($object->GetDBField('CategoryFormat') == 1) || ($event->Special == 'import')) // in one field { $object->setRequired('CategorySeparator', true); $cross_unique_fields[] = 'CategorySeparator'; } - + $ret = $object->Validate(); - + // check if cross unique fields has no same values foreach ($cross_unique_fields as $field_index => $field_name) { if (getArrayValue($object->FieldErrors, $field_name, 'pseudo') == 'required') continue; - + $check_fields = $cross_unique_fields; unset($check_fields[$field_index]); - + foreach ($check_fields as $check_field) { if ($object->GetDBField($field_name) == $object->GetDBField($check_field)) @@ -157,19 +157,19 @@ } } } - + if ($event->Special == 'import') { $this->exportOptions = unserialize($this->Application->RecallVar($event->getPrefixSpecial().'_options')); - + $automatic_fields = ($object->GetDBField('FieldTitles') == 1); $object->setRequired('ExportColumns', !$automatic_fields); $category_prefix = '__CATEGORY__'; if ( $automatic_fields && ($this->exportOptions['SkipFirstRow']) ) { $this->openFile($event); $this->exportOptions['ExportColumns'] = $this->readRecord(); $this->closeFile(); - + // remove additional (non-parseble columns) foreach ($this->exportOptions['ExportColumns'] as $field_index => $field_name) { if (!$this->validateField($field_name, $object)) { @@ -178,7 +178,7 @@ } $category_prefix = ''; } - + // 1. check, that we have column definitions if (!$this->exportOptions['ExportColumns']) { $object->setError('ExportColumns', 'required'); @@ -188,25 +188,25 @@ // 1.1. check that all required fields are present in imported file $missing_columns = Array(); foreach ($object->Fields as $field_name => $field_options) { - if ($object->SkipField($field_name)) continue; - if (getArrayValue($field_options, 'required') && !in_array($field_name, $this->exportOptions['ExportColumns']) ) { + if ($object->SkipField($field_name)) continue; + if (getArrayValue($field_options, 'required') && !in_array($field_name, $this->exportOptions['ExportColumns']) ) { $missing_columns[] = $field_name; $object->setError('ExportColumns', 'required_fields_missing', 'la_error_RequiredColumnsMissing'); $ret = false; } } - + if (!$ret) { $this->Application->Debugger->appendHTML('Missing required for import/export:'); $this->Application->Debugger->dumpVars($missing_columns); } } - - + + // 2. check, that we have only mixed category field or only separated category fields $category_found['mixed'] = false; $category_found['separated'] = false; - + foreach ($this->exportOptions['ExportColumns'] as $import_field) { if (preg_match('/^'.$category_prefix.'Category(Path|[0-9]+)/', $import_field, $rets)) { $category_found[$rets[1] == 'Path' ? 'mixed' : 'separated'] = true; @@ -216,7 +216,7 @@ $object->SetError('ExportColumns', 'unique_category', 'la_error_unique_category_field'); $ret = false; } - + // 3. check, that duplicates check fields are selected & present in imported fields if ($this->exportOptions['ReplaceDuplicates']) { if ($this->exportOptions['CheckDuplicatesMethod'] == 1) { @@ -225,7 +225,7 @@ else { $check_fields = $this->exportOptions['DuplicateCheckFields'] ? explode('|', substr($this->exportOptions['DuplicateCheckFields'], 1, -1)) : Array(); $object =& $event->getObject(); - + $language_id = $this->Application->GetDefaultLanguageId(); foreach ($check_fields as $index => $check_field) { foreach ($object->Fields as $field_name => $field_options) { @@ -237,7 +237,7 @@ } } $this->exportOptions['DuplicateCheckFields'] = $check_fields; - + if (!$check_fields) { $object->setError('CheckDuplicatesMethod', 'required'); $ret = false; @@ -253,10 +253,10 @@ } } } - + return $ret; } - + /** * Returns filename to read import data from * @@ -266,14 +266,14 @@ { if ($this->exportOptions['ImportSource'] == 1) { - $ret = $this->exportOptions['ImportFilename']['name']; + $ret = $this->exportOptions['ImportFilename']; // ['name']; commented by Kostja } else { $ret = $this->exportOptions['ImportLocalFilename']; } return EXPORT_PATH.'/'.$ret; } - + /** * Returns filename to write export data to * @@ -283,7 +283,7 @@ { return EXPORT_PATH.'/'.$this->exportOptions['ExportFilename'].'.'.$this->getFileExtension(); } - + /** * Opens file required for export/import operations * @@ -298,14 +298,14 @@ else { $this->filePointer = fopen($this->getImportFilename(), 'r'); } - + // skip UTF-8 BOM Modifier $first_chars = fread($this->filePointer, 3); if (bin2hex($first_chars) != 'efbbbf') { fseek($this->filePointer, 0); } } - + /** * Closes opened file * @@ -314,7 +314,7 @@ { fclose($this->filePointer); } - + function getExportSQL($count_only = false) { if ($this->exportOptions['export_ids'] === false) @@ -325,7 +325,7 @@ LEFT JOIN '.TABLE_PREFIX.'CategoryItems ci ON ci.ItemResourceId = item_table.ResourceId LEFT JOIN '.TABLE_PREFIX.'Category c ON c.CategoryId = ci.CategoryId WHERE '; - + if ($this->exportOptions['export_cats_ids'][0] == 0) { $sql .= '1'; @@ -336,7 +336,7 @@ } $sql = preg_replace('/(.*) OR $/', '\\1', $sql); } - + $sql .= ' ORDER BY ci.PrimaryCat DESC'; // NEW } else { @@ -345,18 +345,18 @@ FROM '.$this->curItem->TableName.' item_table WHERE '.$this->curItem->IDField.' IN ('.implode(',', $this->exportOptions['export_ids']).')'; } - + if (!$count_only) { $sql .= ' LIMIT '.$this->exportOptions['start_from'].','.EXPORT_STEP; } else { $sql = preg_replace("/^.*SELECT(.*?)FROM(?!_)/is", "SELECT COUNT(*) AS count FROM ", $sql); } - + return $sql; } - + /** * Enter description here... * @@ -367,22 +367,22 @@ $this->exportOptions = unserialize($this->Application->RecallVar($event->getPrefixSpecial().'_options')); $this->exportFields = $this->exportOptions['ExportColumns']; $this->curItem =& $event->getObject( Array('skip_autoload' => true) ); - + $this->openFile($event); - + if ($this->exportOptions['start_from'] == 0) // first export step { if (!getArrayValue($this->exportOptions, 'IsBaseCategory')) { $this->exportOptions['IsBaseCategory'] = 0; } - + if ($this->exportOptions['IsBaseCategory'] ) { $sql = 'SELECT CachedNavbar FROM '.TABLE_PREFIX.'Category WHERE CategoryId = '.$this->Application->GetVar('m_cat_id'); $this->exportOptions['BaseLevel'] = substr_count($this->Conn->GetOne($sql), '>') + 1; // level to cut from other categories } - + // 1. export field titles if required if ($this->exportOptions['IncludeFieldTitles']) { @@ -396,9 +396,9 @@ $this->exportOptions['total_records'] = $this->Conn->GetOne( $this->getExportSQL(true) ); $this->exportOptions['has_custom_fields'] = $this->scanCustomFields(); } - + $this->hasCustomFields = $this->exportOptions['has_custom_fields']; - + // 2. export data $records = $this->Conn->Query( $this->getExportSQL() ); $records_exported = 0; @@ -411,7 +411,7 @@ { $this->loadItemCustomFields(); } - + $data_array = Array(); foreach ($this->exportFields as $export_field) { @@ -421,19 +421,19 @@ $records_exported++; } $this->closeFile(); - + $this->exportOptions['start_from'] += $records_exported; $this->Application->StoreVar($event->getPrefixSpecial().'_options', serialize($this->exportOptions) ); - + return $this->exportOptions; } - + function getItemFields() { // just in case dummy user selected automtic mode & moved columns too :( return array_merge($this->curItem->Fields['AvailableColumns']['options'], $this->curItem->Fields['ExportColumns']['options']); } - + /** * Checks if field really belongs to importable field list * @@ -445,14 +445,14 @@ { // 1. convert custom field $field_name = preg_replace('/^Custom_(.*)/', '__CUSTOM__\\1', $field_name); - + // 2. convert category field (mixed version & serparated version) $field_name = preg_replace('/^Category(Path|[0-9]+)/', '__CATEGORY__Category\\1', $field_name); - + $valid_fields = $object->getPossibleExportColumns(); return isset($valid_fields[$field_name]) || isset($valid_fields['__VIRTUAL__'.$field_name]); } - + /** * Enter description here... * @@ -464,12 +464,12 @@ // load import options in case if not previously loaded in verification function $this->exportOptions = unserialize($this->Application->RecallVar($event->getPrefixSpecial().'_options')); } - + $backup_category_id = $this->Application->GetVar('m_cat_id'); $this->Application->SetVar('m_cat_id', (int)$this->Application->RecallVar('ImportCategory') ); - + $this->openFile($event); - + $bytes_imported = 0; if ($this->exportOptions['start_from'] == 0) // first export step { @@ -479,7 +479,7 @@ $this->exportOptions['start_from'] = ftell($this->filePointer); $bytes_imported = ftell($this->filePointer); } - + $current_category_id = $this->Application->GetVar('m_cat_id'); if ($current_category_id > 0) { $sql = 'SELECT ParentPath FROM '.TABLE_PREFIX.'Category WHERE CategoryId = '.$current_category_id; @@ -495,11 +495,11 @@ } $this->exportFields = $this->exportOptions['ExportColumns']; $this->addToCache('category_parent_path', $this->Application->GetVar('m_cat_id'), $this->exportOptions['ImportCategoryPath']); - + // 2. import data $this->dummyCategory =& $this->Application->recallObject('c.-tmpitem', 'c', Array('skip_autoload' => true)); fseek($this->filePointer, $this->exportOptions['start_from']); - + while (($bytes_imported < IMPORT_CHUNK) && !feof($this->filePointer)) { $this->customValues = Array(); $data = $this->readRecord(); @@ -508,38 +508,38 @@ // set fields used as keys for replace duplicates code $this->resetImportObject($event, IMPORT_TEMP, $data); } - + $this->processCurrentItem($event, $data); } $bytes_imported = ftell($this->filePointer) - $this->exportOptions['start_from']; } - + $this->closeFile(); $this->Application->SetVar('m_cat_id', $backup_category_id); - + $this->exportOptions['start_from'] += $bytes_imported; $this->exportOptions['new_ids_hash'] = getArrayValue($this->cache, 'new_ids'); $this->Application->StoreVar($event->getPrefixSpecial().'_options', serialize($this->exportOptions) ); - + return $this->exportOptions; } - + function setCurrentID() { $this->curItem->setID( $this->curItem->GetDBField($this->curItem->IDField) ); } - + function setFieldValue($field_index, $value) { if (empty($value)) { $value = null; } - + $field_name = $this->exportFields[$field_index]; if ($field_name == 'ResourceId') { return false; } - + if (substr($field_name, 0, 7) == 'Custom_') { $field_name = substr($field_name, 7); $this->customValues[$field_name] = $value; @@ -553,38 +553,38 @@ elseif (substr($field_name, 0, 11) == '__VIRTUAL__') { $field_name = substr($field_name, 11); $this->curItem->SetField($field_name, $value); - } + } else { $this->curItem->SetField($field_name, $value); } } - + function resetImportObject(&$event, $object_type, $record_data = null) { switch ($object_type) { case IMPORT_TEMP: - $this->curItem =& $event->getObject( Array('skip_autoload' => true) ); + $this->curItem =& $event->getObject( Array('skip_autoload' => true) ); break; - + case IMPORT_LIVE: $this->curItem =& $this->Application->recallObject($event->Prefix.'.-tmpitem'.$event->Special, $event->Prefix, Array('skip_autoload' => true)); break; } $this->curItem->Clear(); - + if (isset($record_data)) { $this->setImportData($record_data); } } - + function setImportData($record_data) { foreach ($record_data as $field_index => $field_value) { $this->setFieldValue($field_index, $field_value); } $this->setCurrentID(); } - + /** * Enter description here... * @@ -608,24 +608,24 @@ $load_keys[$key_field] = $this->curItem->GetDBField($key_field); } } - + $this->resetImportObject($event, IMPORT_LIVE); - + if (count($load_keys)) { $where_clause = ''; foreach ($load_keys as $field_name => $field_value) { $where_clause .= '(item_table.`'.$field_name.'` = '.$this->Conn->qstr($field_value).') AND '; } $where_clause = preg_replace('/(.*) AND $/', '\\1', $where_clause); - + $item_id = $this->getFromCache('new_ids', $where_clause); if (!$item_id) { if ($this->exportOptions['CheckDuplicatesMethod'] == 2) { // by other fields $parent_path = $this->getParentPath($category_id); $where_clause = '(c.ParentPath LIKE "'.$parent_path.'%") AND '.$where_clause; } - + $sql = 'SELECT '.$this->curItem->IDField.' FROM '.$this->curItem->TableName.' item_table LEFT JOIN '.TABLE_PREFIX.'CategoryItems ci ON ci.ItemResourceId = item_table.ResourceId @@ -635,34 +635,34 @@ } $save_method = $item_id && $this->curItem->Load($item_id) ? 'Update' : 'Create'; } - + $this->setImportData($record_data); } else { $this->resetImportObject($event, IMPORT_LIVE, $record_data); } - + // create/update categories $backup_category_id = $this->Application->GetVar('m_cat_id'); - + foreach ($this->curItem->CategoryPath as $category_name) { if (!$category_name) continue; $category_id = $this->getFromCache('category_names', $category_name); if ($category_id === false) { // get parent category path to search only in it $current_category_id = $this->Application->GetVar('m_cat_id'); $parent_path = $this->getParentPath($current_category_id); - + // get category id from database by name $sql = 'SELECT CategoryId FROM '.TABLE_PREFIX.'Category WHERE (Name = '.$this->Conn->qstr($category_name).') AND (ParentPath LIKE "'.$parent_path.'%")'; $category_id = $this->Conn->GetOne($sql); - + if ($category_id === false) { // category not in db -> create - $category_fields = Array( 'Name' => $category_name, 'Description' => $category_name, - 'Status' => STATUS_ACTIVE, 'ParentId' => $current_category_id, + $category_fields = Array( 'Name' => $category_name, 'Description' => $category_name, + 'Status' => STATUS_ACTIVE, 'ParentId' => $current_category_id, 'AutomaticFilename' => 1 ); $this->dummyCategory->SetDBFieldsFromHash($category_fields); if ($this->dummyCategory->Create()) { @@ -675,42 +675,42 @@ $this->addToCache('category_names', $category_name, $category_id); } } - + if ($category_id) { $this->Application->SetVar('m_cat_id', $category_id); } } if (!$this->curItem->CategoryPath) { $category_id = $backup_category_id; } - + // create main record if ($save_method == 'Create') { $this->fillRequiredFields($this->false, $this->curItem, true); } - + if (!$this->curItem->$save_method()) { return false; } - + if ($load_keys && ($save_method == 'Create') && $this->exportOptions['ReplaceDuplicates']) { // map new id to old id $this->addToCache('new_ids', $where_clause, $this->curItem->GetID() ); } - + // set custom fields foreach ($this->customValues as $custom_field => $custom_value) { if (($save_method == 'Create') && !$custom_value) continue; $this->curItem->SetCustomField($custom_field, $custom_value); } - + // assign item to categories $this->curItem->assignToCategory($category_id, false); - + $this->Application->SetVar('m_cat_id', $backup_category_id); return true; } - + /** * Returns category parent path, if possible, then from cache * @@ -729,7 +729,7 @@ } return $parent_path; } - + function loadItemCustomFields() { $sql = 'SELECT meta_data.Value, cf.FieldName @@ -738,17 +738,17 @@ WHERE meta_data.ResourceId = '.$this->curItem->GetDBField('ResourceId'); $this->customValues = $this->Conn->GetCol($sql, 'FieldName'); } - + function getFileExtension() { return $this->exportOptions['ExportFormat'] == 1 ? 'csv' : 'xml'; } - + function getLineSeparator($option = 'LineEndings') { return $this->exportOptions[$option] == 1 ? "\r\n" : "\n"; } - + function scanCustomFields() { $ret = false; @@ -763,9 +763,9 @@ } return $ret; } - + /** - * Returns field caption for any exported field + * Returns field caption for any exported field * * @param string $field * @return string @@ -787,10 +787,10 @@ { $ret = $field; } - + return Array($ret); } - + /** * Returns requested field value (including custom fields and category fields) * @@ -814,11 +814,11 @@ { $ret = $this->curItem->GetField($field); } - + $ret = str_replace("\r\n", $this->getLineSeparator('LineEndingsInside'), $ret); return Array($ret); } - + /** * Returns category field(-s) caption based on export mode * @@ -844,7 +844,7 @@ return $ret; } } - + /** * Returns category path in required format for current link * @@ -855,21 +855,21 @@ $category_id = $this->curItem->GetDBField('CategoryId'); $category_path = $this->getFromCache('category_path', $category_id); if (!$category_path) - { + { $sql = 'SELECT CachedNavbar FROM '.TABLE_PREFIX.'Category WHERE CategoryId = '.$category_id; $category_path = $this->Conn->GetOne($sql); $category_path = $category_path ? explode('>', $category_path) : Array(); - + if ($this->exportOptions['IsBaseCategory']) { $i = $this->exportOptions['BaseLevel']; while ($i > 0) { array_shift($category_path); $i--; } } - + $category_count = $this->getMaxCategoryLevel(); if ($this->exportOptions['CategoryFormat'] == 1) { // category path in single field @@ -889,10 +889,10 @@ } $this->addToCache('category_path', $category_id, $category_path); } - + return $category_path; } - + /** * Get maximal category deep level from links beeing exported * @@ -901,18 +901,18 @@ function getMaxCategoryLevel() { static $max_level = -1; - + if ($max_level != -1) { return $max_level; } - + $sql = 'SELECT IF(c.CategoryId IS NULL, 0, MAX( LENGTH(c.ParentPath) - LENGTH( REPLACE(c.ParentPath, "|", "") ) - 1 )) FROM '.$this->curItem->TableName.' item_table LEFT JOIN '.TABLE_PREFIX.'CategoryItems ci ON item_table.ResourceId = ci.ItemResourceId LEFT JOIN '.TABLE_PREFIX.'Category c ON c.CategoryId = ci.CategoryId WHERE (ci.PrimaryCat = 1) AND '; - + $where_clause = ''; if ($this->exportOptions['export_ids'] === false) { // get links from current category & all it's subcategories @@ -932,14 +932,14 @@ } $max_level = $this->Conn->GetOne($sql.'('.$where_clause.')'); - + if ($this->exportOptions['IsBaseCategory'] ) { $max_level -= $this->exportOptions['BaseLevel']; } - + return $max_level; } - + /** * Saves one record to export file * @@ -949,7 +949,7 @@ { fputcsv2($this->filePointer, $fields_hash, $this->exportOptions['FieldsSeparatedBy'], $this->exportOptions['FieldsEnclosedBy'], $this->getLineSeparator() ); } - + function readRecord() { return fgetcsv($this->filePointer, 10000, $this->exportOptions['FieldsSeparatedBy'], $this->exportOptions['FieldsEnclosedBy']);