Index: branches/unlabeled/unlabeled-1.4.10/core/units/categories/categories_item.php =================================================================== diff -u -r3790 -r4039 --- branches/unlabeled/unlabeled-1.4.10/core/units/categories/categories_item.php (.../categories_item.php) (revision 3790) +++ branches/unlabeled/unlabeled-1.4.10/core/units/categories/categories_item.php (.../categories_item.php) (revision 4039) @@ -55,7 +55,7 @@ $sql = 'SELECT ParentPath FROM '.$cat_table.' WHERE CategoryId = %s'; $parent_path = $this->Conn->GetOne( sprintf($sql, $parent_id) ); - if(!$parent_path) $parent_path = '|'.$parent_id.'|'; + if(!$parent_path && $parent_id) $parent_path = '|'.$parent_id.'|'; return $parent_path.$this->GetID().'|'; } Index: branches/unlabeled/unlabeled-1.4.10/kernel/units/categories/categories_item.php =================================================================== diff -u -r3790 -r4039 --- branches/unlabeled/unlabeled-1.4.10/kernel/units/categories/categories_item.php (.../categories_item.php) (revision 3790) +++ branches/unlabeled/unlabeled-1.4.10/kernel/units/categories/categories_item.php (.../categories_item.php) (revision 4039) @@ -55,7 +55,7 @@ $sql = 'SELECT ParentPath FROM '.$cat_table.' WHERE CategoryId = %s'; $parent_path = $this->Conn->GetOne( sprintf($sql, $parent_id) ); - if(!$parent_path) $parent_path = '|'.$parent_id.'|'; + if(!$parent_path && $parent_id) $parent_path = '|'.$parent_id.'|'; return $parent_path.$this->GetID().'|'; } Index: branches/unlabeled/unlabeled-1.8.10/kernel/admin_templates/incs/style.css =================================================================== diff -u -r3667 -r4039 --- branches/unlabeled/unlabeled-1.8.10/kernel/admin_templates/incs/style.css (.../style.css) (revision 3667) +++ branches/unlabeled/unlabeled-1.8.10/kernel/admin_templates/incs/style.css (.../style.css) (revision 4039) @@ -502,4 +502,14 @@ .grid_id_cell TD { padding-right: 2px; +} + +.transparent { + filter: alpha(opacity=50); + -moz-opacity: .5; + opacity: .5; +} + +.none_transparent { + } \ No newline at end of file Index: branches/unlabeled/unlabeled-1.4.2/admin/install/upgrades/inportal_upgrade_v1.1.7.sql =================================================================== diff -u -r3938 -r4039 --- branches/unlabeled/unlabeled-1.4.2/admin/install/upgrades/inportal_upgrade_v1.1.7.sql (.../inportal_upgrade_v1.1.7.sql) (revision 3938) +++ branches/unlabeled/unlabeled-1.4.2/admin/install/upgrades/inportal_upgrade_v1.1.7.sql (.../inportal_upgrade_v1.1.7.sql) (revision 4039) @@ -13,4 +13,6 @@ UPDATE Language SET InputDateFormat = 'm/d/Y', InputTimeFormat = 'g:i:s A'; +CREATE TABLE ImportCache (CacheId int(11) NOT NULL auto_increment, CacheName varchar(255) NOT NULL default '', VarName int(11) NOT NULL default '0', VarValue text NOT NULL, PRIMARY KEY (CacheId), KEY CacheName (CacheName), KEY VarName (VarName)); + UPDATE Modules SET Version = '1.1.7' WHERE Name = 'In-Portal'; \ No newline at end of file Index: branches/unlabeled/unlabeled-1.4.2/kernel/units/general/cat_tag_processor.php =================================================================== diff -u -r3906 -r4039 --- branches/unlabeled/unlabeled-1.4.2/kernel/units/general/cat_tag_processor.php (.../cat_tag_processor.php) (revision 3906) +++ branches/unlabeled/unlabeled-1.4.2/kernel/units/general/cat_tag_processor.php (.../cat_tag_processor.php) (revision 4039) @@ -63,7 +63,10 @@ $cat_object =& $this->Application->recallObject('c', 'c_List'); $sql = 'SELECT CategoryId, ParentId, Name FROM '.$cat_object->TableName.' WHERE CategoryId = '.$params['cat_id']; $res = $this->Conn->GetRow($sql); - + if ($res === false) { + // in case if category is deleted + return ''; + } $block_params['name'] = $params['block']; $block_params['cat_name'] = $res['Name']; $block_params['cat_id'] = $res['CategoryId']; @@ -100,7 +103,31 @@ $special = crc32($parent_cat_id.$types.$except.$recursive); return $special; - } + } + + function ExportStatus($params) + { + define('DBG_SKIP_REPORTING', 1); + + $export_object =& $this->Application->recallObject('CatItemExportHelper'); + + $event = new kEvent($this->getPrefixSpecial().':OnDummy'); + + $action_method = 'perform'.ucfirst($this->Special); + $field_values = $export_object->$action_method($event); + + // finish code is done from JS now + if ($field_values['start_from'] == $field_values['total_records']) + { + if ($this->Special == 'import') { + $this->Application->StoreVar('PermCache_UpdateRequired', 1); + } + } + + $export_options = $export_object->loadOptions($event); + return $export_options['start_from'] * 100 / $export_options['total_records']; + } + } ?> \ No newline at end of file Index: branches/unlabeled/unlabeled-1.41.2/admin/install/inportal_schema.sql =================================================================== diff -u -r3946 -r4039 --- branches/unlabeled/unlabeled-1.41.2/admin/install/inportal_schema.sql (.../inportal_schema.sql) (revision 3946) +++ branches/unlabeled/unlabeled-1.41.2/admin/install/inportal_schema.sql (.../inportal_schema.sql) (revision 4039) @@ -706,4 +706,14 @@ KEY PortalUserId (PortalUserId), KEY AffiliateId (AffiliateId) ) +# -------------------------------------------------------- +CREATE TABLE ImportCache ( + CacheId int(11) NOT NULL auto_increment, + CacheName varchar(255) NOT NULL default '', + VarName int(11) NOT NULL default '0', + VarValue text NOT NULL, + PRIMARY KEY (CacheId), + KEY CacheName (CacheName), + KEY VarName (VarName) +) # -------------------------------------------------------- \ No newline at end of file Index: branches/unlabeled/unlabeled-1.4.2/core/units/general/cat_tag_processor.php =================================================================== diff -u -r3906 -r4039 --- branches/unlabeled/unlabeled-1.4.2/core/units/general/cat_tag_processor.php (.../cat_tag_processor.php) (revision 3906) +++ branches/unlabeled/unlabeled-1.4.2/core/units/general/cat_tag_processor.php (.../cat_tag_processor.php) (revision 4039) @@ -63,7 +63,10 @@ $cat_object =& $this->Application->recallObject('c', 'c_List'); $sql = 'SELECT CategoryId, ParentId, Name FROM '.$cat_object->TableName.' WHERE CategoryId = '.$params['cat_id']; $res = $this->Conn->GetRow($sql); - + if ($res === false) { + // in case if category is deleted + return ''; + } $block_params['name'] = $params['block']; $block_params['cat_name'] = $res['Name']; $block_params['cat_id'] = $res['CategoryId']; @@ -100,7 +103,31 @@ $special = crc32($parent_cat_id.$types.$except.$recursive); return $special; - } + } + + function ExportStatus($params) + { + define('DBG_SKIP_REPORTING', 1); + + $export_object =& $this->Application->recallObject('CatItemExportHelper'); + + $event = new kEvent($this->getPrefixSpecial().':OnDummy'); + + $action_method = 'perform'.ucfirst($this->Special); + $field_values = $export_object->$action_method($event); + + // finish code is done from JS now + if ($field_values['start_from'] == $field_values['total_records']) + { + if ($this->Special == 'import') { + $this->Application->StoreVar('PermCache_UpdateRequired', 1); + } + } + + $export_options = $export_object->loadOptions($event); + return $export_options['start_from'] * 100 / $export_options['total_records']; + } + } ?> \ No newline at end of file Index: branches/unlabeled/unlabeled-1.8.20/admin/index4.php =================================================================== diff -u -r3381 -r4039 --- branches/unlabeled/unlabeled-1.8.20/admin/index4.php (.../index4.php) (revision 3381) +++ branches/unlabeled/unlabeled-1.8.20/admin/index4.php (.../index4.php) (revision 4039) @@ -17,8 +17,7 @@ $end = getmicrotime(); -if ( constOn('DEBUG_MODE') ) -{ +if ($application->isDebugMode() && !dbg_ConstOn('DBG_SKIP_REPORTING')) { echo '

Index: branches/unlabeled/unlabeled-1.1.2/kernel/admin_templates/incs/export.js =================================================================== diff -u --- branches/unlabeled/unlabeled-1.1.2/kernel/admin_templates/incs/export.js (revision 0) +++ branches/unlabeled/unlabeled-1.1.2/kernel/admin_templates/incs/export.js (revision 4039) @@ -0,0 +1,67 @@ +var $http_request = getXMLHTTPObject(); +var $last_responce = new Date(); + +function formatNumber($number, $digits_count) +{ + return parseInt($number) < Math.pow(10, $digits_count - 1) ? str_repeat('0', $digits_count - 1) + $number : $number; +} + +function formatDate($date) +{ + var $hour = formatNumber($date.getHours(), 2); + var $minute = formatNumber($date.getMinutes(), 2); + var $second = formatNumber($date.getSeconds(), 2); + return $hour + ':' + $minute + ':' + $second; +} + +function getSeconds($from_date, $to_date) +{ + var $from_seconds = $from_date.getSeconds() + $from_date.getMinutes() * 60 + $from_date.getHours() * 3600; + var $to_seconds = $to_date.getSeconds() + $to_date.getMinutes() * 60 + $to_date.getHours() * 3600; + return $to_seconds - $from_seconds; +} + +function queryProgress($url, $responce_func) +{ + if ($http_request && $http_request.readyState != 0) { + $http_request.abort(); + } + + $http_request.onreadystatechange = function() { eval($responce_func + '(\'' + $url + '\',\'' + $responce_func + '\');'); }; + $http_request.open('POST', $url, true); + $http_request.setRequestHeader("referer", $url); + $http_request.send('is_xml=1'); + $last_responce = new Date(); +} + +function setProgress($percent) +{ + $percent = parseInt($percent); + document.getElementById('percent_done').style.width = $percent + '%'; + document.getElementById('percent_left').style.width = (100 - $percent) + '%'; + document.getElementById('progress_display').innerHTML = $percent; + document.getElementById('progress_time').innerHTML = '[' + $phrase_LastResponce + ': ' + formatDate($last_responce) + ']'; +} + +function processExportResponce($url, $responce_func) +{ + if ($http_request.readyState == 4) { + if ($http_request.status == 200) { + var $progress = parseFloat($http_request.responseText); + if ($progress) { + setProgress($progress); + if ($progress < 100) { + // proceed to next import step + queryProgress($url, $responce_func); + } + else { + // proceed to finish screen + window.location.href = $finish_template; + } + } + else { + alert($phrase_ResponceError); + } + } + } +} \ No newline at end of file Index: branches/unlabeled/unlabeled-1.11.10/kernel/admin_templates/incs/script.js =================================================================== diff -u -r3625 -r4039 --- branches/unlabeled/unlabeled-1.11.10/kernel/admin_templates/incs/script.js (.../script.js) (revision 3625) +++ branches/unlabeled/unlabeled-1.11.10/kernel/admin_templates/incs/script.js (.../script.js) (revision 4039) @@ -730,4 +730,36 @@ $aSelect.options[$src_num] = $dst_option; $aSelect.options[$dst_num] = $src_option; - } \ No newline at end of file + } + + function getXMLHTTPObject() + { + var http_request = false; + if (window.XMLHttpRequest) { // Mozilla, Safari,... + http_request = new XMLHttpRequest(); + if (http_request.overrideMimeType) { + http_request.overrideMimeType('text/plain'); + // See note below about this line + } + } else if (window.ActiveXObject) { // IE + try { + http_request = new ActiveXObject("Msxml2.XMLHTTP"); + } catch (e) { + try { + http_request = new ActiveXObject("Microsoft.XMLHTTP"); + } catch (e) {} + } + } + return http_request; + } + + function str_repeat($symbol, $count) + { + var $i = 0; + var $ret = ''; + while($i < $count) { + $ret += $symbol; + $i++; + } + return $ret; + } Index: branches/unlabeled/unlabeled-1.4.10/kernel/units/config_general/config_general_tag_processor.php =================================================================== diff -u -r3790 -r4039 --- branches/unlabeled/unlabeled-1.4.10/kernel/units/config_general/config_general_tag_processor.php (.../config_general_tag_processor.php) (revision 3790) +++ branches/unlabeled/unlabeled-1.4.10/kernel/units/config_general/config_general_tag_processor.php (.../config_general_tag_processor.php) (revision 4039) @@ -54,20 +54,21 @@ } } - $block_params['current'] = 1; $block_params['separator'] = $params['separator']; - if($params['cat_id'] == 0) - { + if ($params['cat_id'] == 0) { $block_params['name'] = $params['rootcatblock']; return $this->Application->ParseBlock($block_params); } - else - { + else { $cat_object =& $this->Application->recallObject( $this->getPrefixSpecial(), $this->Prefix.'_List' ); $sql = 'SELECT CategoryId, ParentId, Name FROM '.TABLE_PREFIX.'Category WHERE CategoryId='.$params['cat_id']; $res = $this->Conn->GetRow($sql); + if ($res === false) { + // in case if category is deleted + return ''; + } $block_params['name'] = $params['block']; $block_params['cat_name'] = $res['Name']; $block_params['separator'] = $params['separator']; Index: branches/unlabeled/unlabeled-1.13.10/kernel/units/categories/categories_tag_processor.php =================================================================== diff -u -r3724 -r4039 --- branches/unlabeled/unlabeled-1.13.10/kernel/units/categories/categories_tag_processor.php (.../categories_tag_processor.php) (revision 3724) +++ branches/unlabeled/unlabeled-1.13.10/kernel/units/categories/categories_tag_processor.php (.../categories_tag_processor.php) (revision 4039) @@ -142,7 +142,11 @@ $sql = 'SELECT CategoryId, ParentId, Name FROM '.$this->Application->getUnitOption($this->Prefix, 'TableName').' WHERE CategoryId='.$params['cat_id']; $res = $this->Conn->GetRow($sql); - + if ($res === false) { + // in case if category is deleted + return ''; + } + $block_params['cat_name'] = $res['Name']; $block_params['cat_id'] = $res['CategoryId']; Index: branches/unlabeled/unlabeled-1.13.10/core/units/categories/categories_tag_processor.php =================================================================== diff -u -r3724 -r4039 --- branches/unlabeled/unlabeled-1.13.10/core/units/categories/categories_tag_processor.php (.../categories_tag_processor.php) (revision 3724) +++ branches/unlabeled/unlabeled-1.13.10/core/units/categories/categories_tag_processor.php (.../categories_tag_processor.php) (revision 4039) @@ -142,7 +142,11 @@ $sql = 'SELECT CategoryId, ParentId, Name FROM '.$this->Application->getUnitOption($this->Prefix, 'TableName').' WHERE CategoryId='.$params['cat_id']; $res = $this->Conn->GetRow($sql); - + if ($res === false) { + // in case if category is deleted + return ''; + } + $block_params['cat_name'] = $res['Name']; $block_params['cat_id'] = $res['CategoryId']; Index: branches/unlabeled/unlabeled-1.14.2/kernel/units/general/cat_dbitem_export.php =================================================================== diff -u -r3814 -r4039 --- branches/unlabeled/unlabeled-1.14.2/kernel/units/general/cat_dbitem_export.php (.../cat_dbitem_export.php) (revision 3814) +++ branches/unlabeled/unlabeled-1.14.2/kernel/units/general/cat_dbitem_export.php (.../cat_dbitem_export.php) (revision 4039) @@ -1,7 +1,7 @@ cacheTable = TABLE_PREFIX.'ImportCache'; + } + /** * Returns value from cache if found or false otherwise * @@ -75,12 +91,48 @@ * @param int $key * @param mixed $value */ - function addToCache($type, $key, $value) + function addToCache($type, $key, $value, $is_new = true) { // if (!isset($this->cache[$type])) $this->cache[$type] = Array(); $this->cache[$type][$key] = $value; + if ($is_new) { + $this->cacheStatus[$type][$key] = true; + } } + function storeCache($cache_types) + { + $cache_types = explode(',', $cache_types); + + $values_sql = ''; + foreach ($cache_types as $cache_type) { + $sql_mask = '('.$this->Conn->qstr($cache_type).',%s,%s),'; + $cache = getArrayValue($this->cacheStatus, $cache_type); + if (!$cache) $cache = Array(); + foreach ($cache as $var_name => $cache_status) { + $var_value = $this->cache[$cache_type][$var_name]; + $values_sql .= sprintf($sql_mask, $this->Conn->qstr($var_name), $this->Conn->qstr($var_value) ); + } + } + $values_sql = preg_replace('/(.*),$/', '\\1', $values_sql); + if ($values_sql) { + $sql = 'INSERT INTO '.$this->cacheTable.'(`CacheName`,`VarName`,`VarValue`) VALUES '.$values_sql; + $this->Conn->Query($sql); + } + + } + + function loadCache() + { + $sql = 'SELECT * FROM '.$this->cacheTable; + $records = $this->Conn->Query($sql); + + $this->cache = Array(); + foreach ($records as $record) { + $this->addToCache($record['CacheName'], $record['VarName'], $record['VarValue'], false); + } + } + /** * Fill required fields with dummy values * @@ -160,7 +212,7 @@ if ($event->Special == 'import') { - $this->exportOptions = unserialize($this->Application->RecallVar($event->getPrefixSpecial().'_options')); + $this->exportOptions = $this->loadOptions($event); $automatic_fields = ($object->GetDBField('FieldTitles') == 1); $object->setRequired('ExportColumns', !$automatic_fields); @@ -252,6 +304,7 @@ } } } + $this->saveOptions($event); } return $ret; @@ -364,7 +417,7 @@ */ function performExport(&$event) { - $this->exportOptions = unserialize($this->Application->RecallVar($event->getPrefixSpecial().'_options')); + $this->exportOptions = $this->loadOptions($event); $this->exportFields = $this->exportOptions['ExportColumns']; $this->curItem =& $event->getObject( Array('skip_autoload' => true) ); @@ -423,7 +476,7 @@ $this->closeFile(); $this->exportOptions['start_from'] += $records_exported; - $this->Application->StoreVar($event->getPrefixSpecial().'_options', serialize($this->exportOptions) ); + $this->saveOptions($event); return $this->exportOptions; } @@ -462,7 +515,7 @@ { if (!$this->exportOptions) { // load import options in case if not previously loaded in verification function - $this->exportOptions = unserialize($this->Application->RecallVar($event->getPrefixSpecial().'_options')); + $this->exportOptions = $this->loadOptions($event); } $backup_category_id = $this->Application->GetVar('m_cat_id'); @@ -491,8 +544,9 @@ $this->exportOptions['total_records'] = filesize($this->getImportFilename()); } else { - $this->cache['new_ids'] = $this->exportOptions['new_ids_hash']; + $this->loadCache(); } + $this->exportFields = $this->exportOptions['ExportColumns']; $this->addToCache('category_parent_path', $this->Application->GetVar('m_cat_id'), $this->exportOptions['ImportCategoryPath']); @@ -518,9 +572,14 @@ $this->Application->SetVar('m_cat_id', $backup_category_id); $this->exportOptions['start_from'] += $bytes_imported; - $this->exportOptions['new_ids_hash'] = getArrayValue($this->cache, 'new_ids'); - $this->Application->StoreVar($event->getPrefixSpecial().'_options', serialize($this->exportOptions) ); - + $this->storeCache('new_ids'); + + $this->saveOptions($event); + + if ($this->exportOptions['start_from'] == $this->exportOptions['total_records']) { + $this->Conn->Query('TRUNCATE TABLE '.$this->cacheTable); + } + return $this->exportOptions; } @@ -535,7 +594,7 @@ $value = null; } - $field_name = $this->exportFields[$field_index]; + $field_name = getArrayValue($this->exportFields, $field_index); if ($field_name == 'ResourceId') { return false; } @@ -591,6 +650,54 @@ $this->setCurrentID(); } + + function getItemCategory() + { + $backup_category_id = $this->Application->GetVar('m_cat_id'); + foreach ($this->curItem->CategoryPath as $category_index => $category_name) { + if (!$category_name) continue; + $category_key = crc32( implode(':', array_slice($this->curItem->CategoryPath, 0, $category_index + 1) ) ); + + $category_id = $this->getFromCache('category_names', $category_key); + if ($category_id === false) { + // get parent category path to search only in it + $current_category_id = $this->Application->GetVar('m_cat_id'); + $parent_path = $this->getParentPath($current_category_id); + + // get category id from database by name + $sql = 'SELECT CategoryId + FROM '.TABLE_PREFIX.'Category + WHERE (Name = '.$this->Conn->qstr($category_name).') AND (ParentPath LIKE "'.$parent_path.'%")'; + $category_id = $this->Conn->GetOne($sql); + + if ($category_id === false) { + // category not in db -> create + $category_fields = Array( 'Name' => $category_name, 'Description' => $category_name, + 'Status' => STATUS_ACTIVE, 'ParentId' => $current_category_id, 'AutomaticFilename' => 1 + ); + $this->dummyCategory->SetDBFieldsFromHash($category_fields); + if ($this->dummyCategory->Create()) { + $category_id = $this->dummyCategory->GetID(); + $this->addToCache('category_parent_path', $category_id, $this->dummyCategory->GetDBField('ParentPath')); + $this->addToCache('category_names', $category_key, $category_id); + } + } + else { + $this->addToCache('category_names', $category_key, $category_id); + } + } + + if ($category_id) { + $this->Application->SetVar('m_cat_id', $category_id); + } + } + if (!$this->curItem->CategoryPath) { + $category_id = $backup_category_id; + } + + return $category_id; + } + /** * Enter description here... * @@ -599,10 +706,15 @@ function processCurrentItem(&$event, $record_data) { $save_method = 'Create'; + $load_keys = Array(); + + // create/update categories + $backup_category_id = $this->Application->GetVar('m_cat_id'); + // perform replace duplicates code if ($this->exportOptions['ReplaceDuplicates']) { // get replace keys first, then reset current item to empty one - $load_keys = Array(); + $category_id = $this->getItemCategory(); if ($this->exportOptions['CheckDuplicatesMethod'] == 1) { if ($this->curItem->GetID()) { $load_keys = Array($this->curItem->IDField => $this->curItem->GetID()); @@ -624,7 +736,7 @@ } $where_clause = preg_replace('/(.*) AND $/', '\\1', $where_clause); - $item_id = $this->getFromCache('new_ids', $where_clause); + $item_id = $this->getFromCache('new_ids', crc32($where_clause)); if (!$item_id) { if ($this->exportOptions['CheckDuplicatesMethod'] == 2) { // by other fields @@ -646,64 +758,24 @@ } else { $this->resetImportObject($event, IMPORT_LIVE, $record_data); + $category_id = $this->getItemCategory(); } - - // create/update categories - $backup_category_id = $this->Application->GetVar('m_cat_id'); - foreach ($this->curItem->CategoryPath as $category_index => $category_name) { - if (!$category_name) continue; - $category_key = crc32( implode(':', array_slice($this->curItem->CategoryPath, 0, $category_index + 1) ) ); - - $category_id = $this->getFromCache('category_names', $category_key); - if ($category_id === false) { - // get parent category path to search only in it - $current_category_id = $this->Application->GetVar('m_cat_id'); - $parent_path = $this->getParentPath($current_category_id); - - // get category id from database by name - $sql = 'SELECT CategoryId - FROM '.TABLE_PREFIX.'Category - WHERE (Name = '.$this->Conn->qstr($category_name).') AND (ParentPath LIKE "'.$parent_path.'%")'; - $category_id = $this->Conn->GetOne($sql); - - if ($category_id === false) { - // category not in db -> create - $category_fields = Array( 'Name' => $category_name, 'Description' => $category_name, - 'Status' => STATUS_ACTIVE, 'ParentId' => $current_category_id, 'AutomaticFilename' => 1 - ); - $this->dummyCategory->SetDBFieldsFromHash($category_fields); - if ($this->dummyCategory->Create()) { - $category_id = $this->dummyCategory->GetID(); - $this->addToCache('category_parent_path', $category_id, $this->dummyCategory->GetDBField('ParentPath')); - $this->addToCache('category_names', $category_key, $category_id); - } - } - else { - $this->addToCache('category_names', $category_key, $category_id); - } - } - - if ($category_id) { - $this->Application->SetVar('m_cat_id', $category_id); - } - } - if (!$this->curItem->CategoryPath) { - $category_id = $backup_category_id; - } - // create main record if ($save_method == 'Create') { $this->fillRequiredFields($this->false, $this->curItem, true); } +// $sql_start = getmicrotime(); if (!$this->curItem->$save_method()) { return false; } - +// $sql_end = getmicrotime(); +// $this->saveLog('SQL ['.$save_method.'] Time: '.($sql_end - $sql_start).'s'); + if ($load_keys && ($save_method == 'Create') && $this->exportOptions['ReplaceDuplicates']) { // map new id to old id - $this->addToCache('new_ids', $where_clause, $this->curItem->GetID() ); + $this->addToCache('new_ids', crc32($where_clause), $this->curItem->GetID() ); } // set custom fields @@ -719,6 +791,13 @@ return true; } + /*function saveLog($msg) + { + $fp = fopen(FULL_PATH.'/sqls.log', 'a'); + fwrite($fp, $msg."\n"); + fclose($fp); + }*/ + /** * Returns category parent path, if possible, then from cache * @@ -962,6 +1041,19 @@ { return fgetcsv($this->filePointer, 10000, $this->exportOptions['FieldsSeparatedBy'], $this->exportOptions['FieldsEnclosedBy']); } + + function saveOptions(&$event, $options = null) + { + if (!isset($options)) { + $options = $this->exportOptions; + } + $this->Application->StoreVar($event->getPrefixSpecial().'_options', serialize($options) ); + } + + function loadOptions(&$event) + { + return unserialize($this->Application->RecallVar($event->getPrefixSpecial().'_options')); + } } ?> Index: branches/unlabeled/unlabeled-1.17.20/index.php =================================================================== diff -u -r3308 -r4039 --- branches/unlabeled/unlabeled-1.17.20/index.php (.../index.php) (revision 3308) +++ branches/unlabeled/unlabeled-1.17.20/index.php (.../index.php) (revision 4039) @@ -16,8 +16,7 @@ $end = getmicrotime(); -if ( $application->isDebugMode() ) -{ +if ($application->isDebugMode() && !dbg_ConstOn('DBG_SKIP_REPORTING')) { echo '

Memory used: '.round(memory_get_usage()/1024/1024, 1).' Mb
'; echo 'Time used: '.round(($end - $start), 5).' Sec
'; } Index: branches/unlabeled/unlabeled-1.14.2/core/units/general/cat_dbitem_export.php =================================================================== diff -u -r3814 -r4039 --- branches/unlabeled/unlabeled-1.14.2/core/units/general/cat_dbitem_export.php (.../cat_dbitem_export.php) (revision 3814) +++ branches/unlabeled/unlabeled-1.14.2/core/units/general/cat_dbitem_export.php (.../cat_dbitem_export.php) (revision 4039) @@ -1,7 +1,7 @@ cacheTable = TABLE_PREFIX.'ImportCache'; + } + /** * Returns value from cache if found or false otherwise * @@ -75,12 +91,48 @@ * @param int $key * @param mixed $value */ - function addToCache($type, $key, $value) + function addToCache($type, $key, $value, $is_new = true) { // if (!isset($this->cache[$type])) $this->cache[$type] = Array(); $this->cache[$type][$key] = $value; + if ($is_new) { + $this->cacheStatus[$type][$key] = true; + } } + function storeCache($cache_types) + { + $cache_types = explode(',', $cache_types); + + $values_sql = ''; + foreach ($cache_types as $cache_type) { + $sql_mask = '('.$this->Conn->qstr($cache_type).',%s,%s),'; + $cache = getArrayValue($this->cacheStatus, $cache_type); + if (!$cache) $cache = Array(); + foreach ($cache as $var_name => $cache_status) { + $var_value = $this->cache[$cache_type][$var_name]; + $values_sql .= sprintf($sql_mask, $this->Conn->qstr($var_name), $this->Conn->qstr($var_value) ); + } + } + $values_sql = preg_replace('/(.*),$/', '\\1', $values_sql); + if ($values_sql) { + $sql = 'INSERT INTO '.$this->cacheTable.'(`CacheName`,`VarName`,`VarValue`) VALUES '.$values_sql; + $this->Conn->Query($sql); + } + + } + + function loadCache() + { + $sql = 'SELECT * FROM '.$this->cacheTable; + $records = $this->Conn->Query($sql); + + $this->cache = Array(); + foreach ($records as $record) { + $this->addToCache($record['CacheName'], $record['VarName'], $record['VarValue'], false); + } + } + /** * Fill required fields with dummy values * @@ -160,7 +212,7 @@ if ($event->Special == 'import') { - $this->exportOptions = unserialize($this->Application->RecallVar($event->getPrefixSpecial().'_options')); + $this->exportOptions = $this->loadOptions($event); $automatic_fields = ($object->GetDBField('FieldTitles') == 1); $object->setRequired('ExportColumns', !$automatic_fields); @@ -252,6 +304,7 @@ } } } + $this->saveOptions($event); } return $ret; @@ -364,7 +417,7 @@ */ function performExport(&$event) { - $this->exportOptions = unserialize($this->Application->RecallVar($event->getPrefixSpecial().'_options')); + $this->exportOptions = $this->loadOptions($event); $this->exportFields = $this->exportOptions['ExportColumns']; $this->curItem =& $event->getObject( Array('skip_autoload' => true) ); @@ -423,7 +476,7 @@ $this->closeFile(); $this->exportOptions['start_from'] += $records_exported; - $this->Application->StoreVar($event->getPrefixSpecial().'_options', serialize($this->exportOptions) ); + $this->saveOptions($event); return $this->exportOptions; } @@ -462,7 +515,7 @@ { if (!$this->exportOptions) { // load import options in case if not previously loaded in verification function - $this->exportOptions = unserialize($this->Application->RecallVar($event->getPrefixSpecial().'_options')); + $this->exportOptions = $this->loadOptions($event); } $backup_category_id = $this->Application->GetVar('m_cat_id'); @@ -491,8 +544,9 @@ $this->exportOptions['total_records'] = filesize($this->getImportFilename()); } else { - $this->cache['new_ids'] = $this->exportOptions['new_ids_hash']; + $this->loadCache(); } + $this->exportFields = $this->exportOptions['ExportColumns']; $this->addToCache('category_parent_path', $this->Application->GetVar('m_cat_id'), $this->exportOptions['ImportCategoryPath']); @@ -518,9 +572,14 @@ $this->Application->SetVar('m_cat_id', $backup_category_id); $this->exportOptions['start_from'] += $bytes_imported; - $this->exportOptions['new_ids_hash'] = getArrayValue($this->cache, 'new_ids'); - $this->Application->StoreVar($event->getPrefixSpecial().'_options', serialize($this->exportOptions) ); - + $this->storeCache('new_ids'); + + $this->saveOptions($event); + + if ($this->exportOptions['start_from'] == $this->exportOptions['total_records']) { + $this->Conn->Query('TRUNCATE TABLE '.$this->cacheTable); + } + return $this->exportOptions; } @@ -535,7 +594,7 @@ $value = null; } - $field_name = $this->exportFields[$field_index]; + $field_name = getArrayValue($this->exportFields, $field_index); if ($field_name == 'ResourceId') { return false; } @@ -591,6 +650,54 @@ $this->setCurrentID(); } + + function getItemCategory() + { + $backup_category_id = $this->Application->GetVar('m_cat_id'); + foreach ($this->curItem->CategoryPath as $category_index => $category_name) { + if (!$category_name) continue; + $category_key = crc32( implode(':', array_slice($this->curItem->CategoryPath, 0, $category_index + 1) ) ); + + $category_id = $this->getFromCache('category_names', $category_key); + if ($category_id === false) { + // get parent category path to search only in it + $current_category_id = $this->Application->GetVar('m_cat_id'); + $parent_path = $this->getParentPath($current_category_id); + + // get category id from database by name + $sql = 'SELECT CategoryId + FROM '.TABLE_PREFIX.'Category + WHERE (Name = '.$this->Conn->qstr($category_name).') AND (ParentPath LIKE "'.$parent_path.'%")'; + $category_id = $this->Conn->GetOne($sql); + + if ($category_id === false) { + // category not in db -> create + $category_fields = Array( 'Name' => $category_name, 'Description' => $category_name, + 'Status' => STATUS_ACTIVE, 'ParentId' => $current_category_id, 'AutomaticFilename' => 1 + ); + $this->dummyCategory->SetDBFieldsFromHash($category_fields); + if ($this->dummyCategory->Create()) { + $category_id = $this->dummyCategory->GetID(); + $this->addToCache('category_parent_path', $category_id, $this->dummyCategory->GetDBField('ParentPath')); + $this->addToCache('category_names', $category_key, $category_id); + } + } + else { + $this->addToCache('category_names', $category_key, $category_id); + } + } + + if ($category_id) { + $this->Application->SetVar('m_cat_id', $category_id); + } + } + if (!$this->curItem->CategoryPath) { + $category_id = $backup_category_id; + } + + return $category_id; + } + /** * Enter description here... * @@ -599,10 +706,15 @@ function processCurrentItem(&$event, $record_data) { $save_method = 'Create'; + $load_keys = Array(); + + // create/update categories + $backup_category_id = $this->Application->GetVar('m_cat_id'); + // perform replace duplicates code if ($this->exportOptions['ReplaceDuplicates']) { // get replace keys first, then reset current item to empty one - $load_keys = Array(); + $category_id = $this->getItemCategory(); if ($this->exportOptions['CheckDuplicatesMethod'] == 1) { if ($this->curItem->GetID()) { $load_keys = Array($this->curItem->IDField => $this->curItem->GetID()); @@ -624,7 +736,7 @@ } $where_clause = preg_replace('/(.*) AND $/', '\\1', $where_clause); - $item_id = $this->getFromCache('new_ids', $where_clause); + $item_id = $this->getFromCache('new_ids', crc32($where_clause)); if (!$item_id) { if ($this->exportOptions['CheckDuplicatesMethod'] == 2) { // by other fields @@ -646,64 +758,24 @@ } else { $this->resetImportObject($event, IMPORT_LIVE, $record_data); + $category_id = $this->getItemCategory(); } - - // create/update categories - $backup_category_id = $this->Application->GetVar('m_cat_id'); - foreach ($this->curItem->CategoryPath as $category_index => $category_name) { - if (!$category_name) continue; - $category_key = crc32( implode(':', array_slice($this->curItem->CategoryPath, 0, $category_index + 1) ) ); - - $category_id = $this->getFromCache('category_names', $category_key); - if ($category_id === false) { - // get parent category path to search only in it - $current_category_id = $this->Application->GetVar('m_cat_id'); - $parent_path = $this->getParentPath($current_category_id); - - // get category id from database by name - $sql = 'SELECT CategoryId - FROM '.TABLE_PREFIX.'Category - WHERE (Name = '.$this->Conn->qstr($category_name).') AND (ParentPath LIKE "'.$parent_path.'%")'; - $category_id = $this->Conn->GetOne($sql); - - if ($category_id === false) { - // category not in db -> create - $category_fields = Array( 'Name' => $category_name, 'Description' => $category_name, - 'Status' => STATUS_ACTIVE, 'ParentId' => $current_category_id, 'AutomaticFilename' => 1 - ); - $this->dummyCategory->SetDBFieldsFromHash($category_fields); - if ($this->dummyCategory->Create()) { - $category_id = $this->dummyCategory->GetID(); - $this->addToCache('category_parent_path', $category_id, $this->dummyCategory->GetDBField('ParentPath')); - $this->addToCache('category_names', $category_key, $category_id); - } - } - else { - $this->addToCache('category_names', $category_key, $category_id); - } - } - - if ($category_id) { - $this->Application->SetVar('m_cat_id', $category_id); - } - } - if (!$this->curItem->CategoryPath) { - $category_id = $backup_category_id; - } - // create main record if ($save_method == 'Create') { $this->fillRequiredFields($this->false, $this->curItem, true); } +// $sql_start = getmicrotime(); if (!$this->curItem->$save_method()) { return false; } - +// $sql_end = getmicrotime(); +// $this->saveLog('SQL ['.$save_method.'] Time: '.($sql_end - $sql_start).'s'); + if ($load_keys && ($save_method == 'Create') && $this->exportOptions['ReplaceDuplicates']) { // map new id to old id - $this->addToCache('new_ids', $where_clause, $this->curItem->GetID() ); + $this->addToCache('new_ids', crc32($where_clause), $this->curItem->GetID() ); } // set custom fields @@ -719,6 +791,13 @@ return true; } + /*function saveLog($msg) + { + $fp = fopen(FULL_PATH.'/sqls.log', 'a'); + fwrite($fp, $msg."\n"); + fclose($fp); + }*/ + /** * Returns category parent path, if possible, then from cache * @@ -962,6 +1041,19 @@ { return fgetcsv($this->filePointer, 10000, $this->exportOptions['FieldsSeparatedBy'], $this->exportOptions['FieldsEnclosedBy']); } + + function saveOptions(&$event, $options = null) + { + if (!isset($options)) { + $options = $this->exportOptions; + } + $this->Application->StoreVar($event->getPrefixSpecial().'_options', serialize($options) ); + } + + function loadOptions(&$event) + { + return unserialize($this->Application->RecallVar($event->getPrefixSpecial().'_options')); + } } ?> Index: branches/unlabeled/unlabeled-1.72.10/admin/install/langpacks/english.lang =================================================================== diff -u -r3956 -r4039 --- branches/unlabeled/unlabeled-1.72.10/admin/install/langpacks/english.lang (.../english.lang) (revision 3956) +++ branches/unlabeled/unlabeled-1.72.10/admin/install/langpacks/english.lang (.../english.lang) (revision 4039) @@ -246,6 +246,7 @@ U3VnZ2VzdCB0byBhIGZyaWVuZA==VXNlciB1bnN1YnNjcmliZWQ=VmFsaWRhdGUgVXNlcg== + QW4gZXJyb3IgaGFzIG9jY3VyZWQgZHVyaW5nIGV4cG9ydC4=QWR2YW5jZWQgQ1NTQXV0b21hdGljIEZpbGVuYW1lQXZhaWxhYmxlIENvbHVtbnM= @@ -356,6 +357,7 @@ VGhpcyBoZWxwIHNlY3Rpb24gZG9lcyBub3QgeWV0IGV4aXN0LCBpdCdzIGNvbWluZyBzb29uIQ==aHRtbA==SUQgRmllbGQ= + QW4gZXJyb3IgaGFzIG9jY3VyZWQgZHVyaW5nIGltcG9ydC4=SW1wb3J0aW5nIEVtYWlsIEV2ZW50cyAuLi4=SW1wb3J0aW5nIExhbmd1YWdlcyAuLi4=SW1wb3J0aW5nIFBocmFzZXMgLi4u @@ -371,6 +373,7 @@ S0I=TGFuZ3VhZ2U=SW1wb3J0IHByb2dyZXNz + TGFzdCBSZXNwb25jZQ==TGFzdCBVcGRhdGVkRGF0ZQ==TGluayBEZXNjcmlwdGlvbg== @@ -1918,14 +1921,17 @@ cG9zdHMgdXBkYXRlZA==UG93ZXJlZCBieQ==Q2l0eQ== + Q29tcGFueQ==Q291bnRyeQ==QmlydGhkYXRlRS1tYWls + RmF4Rmlyc3QgTmFtZQ==TGFzdCBOYW1lUGhvbmU=U3RhdGU=U3RyZWV0 + U3RyZWV0IDI=WmlwUHJpdmFjeQ==UHJpdmF0ZSBtZXNzYWdlcyB1cGRhdGVk Index: branches/unlabeled/unlabeled-1.35.2/kernel/units/general/cat_event_handler.php =================================================================== diff -u -r3853 -r4039 --- branches/unlabeled/unlabeled-1.35.2/kernel/units/general/cat_event_handler.php (.../cat_event_handler.php) (revision 3853) +++ branches/unlabeled/unlabeled-1.35.2/kernel/units/general/cat_event_handler.php (.../cat_event_handler.php) (revision 4039) @@ -1396,11 +1396,12 @@ $field_values['ExportColumns'] = $field_values['ExportColumns'] ? explode('|', substr($field_values['ExportColumns'], 1, -1) ) : Array(); $field_values['start_from'] = 0; - $this->Application->StoreVar($event->getPrefixSpecial().'_options', serialize($field_values) ); - + $export_object->saveOptions($event, $field_values); + if( $export_object->verifyOptions($event) ) { - $this->doExport($event); + $event->redirect = $this->getModuleFolder($event).'/'.$event->Special.'_progress'; + } else { @@ -1410,63 +1411,6 @@ } /** - * Enter description here... - * - * @param kEvent $event - */ - function doExport(&$event) - { - if ($event->Name == 'OnExportBegin') - { - $done_percent = 0; - } - else { - $export_options = unserialize($this->Application->RecallVar($event->getPrefixSpecial().'_options')); - $done_percent = round($export_options['start_from'] * 100 / $export_options['total_records'], 0); - } - - $block_params = Array( 'name' => $this->getModuleFolder($event).'/'.$event->Special.'_progress', - 'percent_done' => $done_percent, - 'percent_left' => 100 - $done_percent); - - $this->Application->InitParser(); - $this->Application->setUnitOption($event->Prefix, 'AutoLoad', false); - echo $this->Application->ParseBlock($block_params); - flush(); - - $export_object =& $this->Application->recallObject('CatItemExportHelper'); - - $action_method = 'perform'.ucfirst($event->Special); - $field_values = $export_object->$action_method($event); - - if ($field_values['start_from'] == $field_values['total_records']) - { - if ($event->Special == 'import') { - $this->Application->StoreVar('PermCache_UpdateRequired', 1); - $event->SetRedirectParam('index_file', 'category/category_maint.php'); - } - else { - $event->redirect = $this->getModuleFolder($event).'/'.$event->Special.'_finish'; - } - } - else { - $event->redirect = $this->getModuleFolder($event).'/'.$event->Special.'_progress'; - $redirect_params = Array($event->getPrefixSpecial().'_event' => 'OnExportProgress', 'pass' => 'm,'.$event->getPrefixSpecial(), 'rand' => rand(0,1000000)); - $event->setRedirectParams($redirect_params); - } - } - - /** - * Next export steps - * - * @param kEvent $event - */ - function OnExportProgress(&$event) - { - $this->doExport($event); - } - - /** * Sets correct available & export fields * * @param kEvent $event Index: branches/unlabeled/unlabeled-1.35.2/core/units/general/cat_event_handler.php =================================================================== diff -u -r3853 -r4039 --- branches/unlabeled/unlabeled-1.35.2/core/units/general/cat_event_handler.php (.../cat_event_handler.php) (revision 3853) +++ branches/unlabeled/unlabeled-1.35.2/core/units/general/cat_event_handler.php (.../cat_event_handler.php) (revision 4039) @@ -1396,11 +1396,12 @@ $field_values['ExportColumns'] = $field_values['ExportColumns'] ? explode('|', substr($field_values['ExportColumns'], 1, -1) ) : Array(); $field_values['start_from'] = 0; - $this->Application->StoreVar($event->getPrefixSpecial().'_options', serialize($field_values) ); - + $export_object->saveOptions($event, $field_values); + if( $export_object->verifyOptions($event) ) { - $this->doExport($event); + $event->redirect = $this->getModuleFolder($event).'/'.$event->Special.'_progress'; + } else { @@ -1410,63 +1411,6 @@ } /** - * Enter description here... - * - * @param kEvent $event - */ - function doExport(&$event) - { - if ($event->Name == 'OnExportBegin') - { - $done_percent = 0; - } - else { - $export_options = unserialize($this->Application->RecallVar($event->getPrefixSpecial().'_options')); - $done_percent = round($export_options['start_from'] * 100 / $export_options['total_records'], 0); - } - - $block_params = Array( 'name' => $this->getModuleFolder($event).'/'.$event->Special.'_progress', - 'percent_done' => $done_percent, - 'percent_left' => 100 - $done_percent); - - $this->Application->InitParser(); - $this->Application->setUnitOption($event->Prefix, 'AutoLoad', false); - echo $this->Application->ParseBlock($block_params); - flush(); - - $export_object =& $this->Application->recallObject('CatItemExportHelper'); - - $action_method = 'perform'.ucfirst($event->Special); - $field_values = $export_object->$action_method($event); - - if ($field_values['start_from'] == $field_values['total_records']) - { - if ($event->Special == 'import') { - $this->Application->StoreVar('PermCache_UpdateRequired', 1); - $event->SetRedirectParam('index_file', 'category/category_maint.php'); - } - else { - $event->redirect = $this->getModuleFolder($event).'/'.$event->Special.'_finish'; - } - } - else { - $event->redirect = $this->getModuleFolder($event).'/'.$event->Special.'_progress'; - $redirect_params = Array($event->getPrefixSpecial().'_event' => 'OnExportProgress', 'pass' => 'm,'.$event->getPrefixSpecial(), 'rand' => rand(0,1000000)); - $event->setRedirectParams($redirect_params); - } - } - - /** - * Next export steps - * - * @param kEvent $event - */ - function OnExportProgress(&$event) - { - $this->doExport($event); - } - - /** * Sets correct available & export fields * * @param kEvent $event