Index: branches/5.2.x/core/units/helpers/category_helper.php =================================================================== diff -u -N -r15137 -r15226 --- branches/5.2.x/core/units/helpers/category_helper.php (.../category_helper.php) (revision 15137) +++ branches/5.2.x/core/units/helpers/category_helper.php (.../category_helper.php) (revision 15226) @@ -1,6 +1,6 @@ Application->isCachingType(CACHING_TYPE_MEMORY) ) { - $this->Application->rebuildCache('master:StructureTree', kCache::REBUILD_NOW, CacheSettings::$structureTreeRebuildTime); - } - else { - $this->Application->rebuildDBCache('StructureTree', kCache::REBUILD_NOW, CacheSettings::$structureTreeRebuildTime); - } - // generate structure tree from scratch $ml_helper = $this->Application->recallObject('kMultiLanguageHelper'); /* @var $ml_helper kMultiLanguageHelper */ @@ -525,13 +518,6 @@ return unserialize($data); } - if ( $this->Application->isCachingType(CACHING_TYPE_MEMORY) ) { - $this->Application->rebuildCache('master:template_mapping', kCache::REBUILD_NOW, CacheSettings::$templateMappingRebuildTime); - } - else { - $this->Application->rebuildDBCache('template_mapping', kCache::REBUILD_NOW, CacheSettings::$templateMappingRebuildTime); - } - $sql = 'SELECT IF(c.`Type` = ' . PAGE_TYPE_TEMPLATE . ', CONCAT(c.Template, ":", c.ThemeId), CONCAT("id:", c.CategoryId)) AS SrcTemplate, LOWER( Index: branches/5.2.x/core/units/helpers/sections_helper.php =================================================================== diff -u -N -r14668 -r15226 --- branches/5.2.x/core/units/helpers/sections_helper.php (.../sections_helper.php) (revision 14668) +++ branches/5.2.x/core/units/helpers/sections_helper.php (.../sections_helper.php) (revision 15226) @@ -1,6 +1,6 @@ Application->isCachingType(CACHING_TYPE_MEMORY) ) { - $this->Application->rebuildCache('master:sections_parsed', kCache::REBUILD_NOW, CacheSettings::$sectionsParsedRebuildTime); - } - else { - $this->Application->rebuildDBCache('sections_parsed', kCache::REBUILD_NOW, CacheSettings::$sectionsParsedRebuildTime); - } - if ( !defined('IS_INSTALL') || !IS_INSTALL ) { // don't reread all configs during install, because they are reread on every install step $this->Application->UnitConfigReader->ReReadConfigs(); Index: branches/5.2.x/core/kernel/managers/cache_manager.php =================================================================== diff -u -N -r15137 -r15226 --- branches/5.2.x/core/kernel/managers/cache_manager.php (.../cache_manager.php) (revision 15137) +++ branches/5.2.x/core/kernel/managers/cache_manager.php (.../cache_manager.php) (revision 15226) @@ -1,6 +1,6 @@ Application->isCachingType(CACHING_TYPE_MEMORY) ) { - $this->Application->rebuildCache('master:configs_parsed', kCache::REBUILD_NOW, CacheSettings::$unitCacheRebuildTime); - } - else { - $this->Application->rebuildDBCache('configs_parsed', kCache::REBUILD_NOW, CacheSettings::$unitCacheRebuildTime); - } - return false; } @@ -402,7 +395,7 @@ $this->Application->rebuildCache('master:configs_parsed', kCache::REBUILD_LATER, CacheSettings::$unitCacheRebuildTime); } else { - $this->Application->rebuildDBCache('configs_parsed', kCache::REBUILD_LATER, CacheSettings::$unitCacheRebuildTime); + $this->rebuildDBCache('configs_parsed', kCache::REBUILD_LATER, CacheSettings::$unitCacheRebuildTime); } } @@ -418,7 +411,7 @@ $this->Application->rebuildCache('master:sections_parsed', kCache::REBUILD_LATER, CacheSettings::$sectionsParsedRebuildTime); } else { - $this->Application->rebuildDBCache('sections_parsed', kCache::REBUILD_LATER, CacheSettings::$sectionsParsedRebuildTime); + $this->rebuildDBCache('sections_parsed', kCache::REBUILD_LATER, CacheSettings::$sectionsParsedRebuildTime); } } @@ -502,16 +495,6 @@ } /** - * Prints caching statistics - * - * @access public - */ - public function printStatistics() - { - $this->cacheHandler->printStatistics(); - } - - /** * Returns cached $key value from cache named $cache_name * * @param int $key key name from cache @@ -526,7 +509,7 @@ } /** - * Adds new value to cache $cache_name and identified by key $key + * Stores new $value in cache with $key name * * @param int $key key name to add to cache * @param mixed $value value of cached record @@ -540,17 +523,31 @@ } /** + * Stores new $value in cache with $key name (only if not there already) + * + * @param int $key key name to add to cache + * @param mixed $value value of cached record + * @param int $expiration when value expires (0 - doesn't expire) + * @return bool + * @access public + */ + public function addCache($key, $value, $expiration = 0) + { + return $this->cacheHandler->addCache($key, $value, $expiration); + } + + /** * Sets rebuilding mode for given cache * * @param string $name * @param int $mode * @param int $max_rebuilding_time - * @return void + * @return bool * @access public */ public function rebuildCache($name, $mode = null, $max_rebuilding_time = 0) { - $this->cacheHandler->rebuildCache($name, $mode, $max_rebuilding_time); + return $this->cacheHandler->rebuildCache($name, $mode, $max_rebuilding_time); } /** @@ -587,6 +584,7 @@ public function getDBCache($name, $max_rebuild_seconds = 0) { // no serials in cache key OR cache is outdated + $rebuilding = false; $wait_seconds = $max_rebuild_seconds; while (true) { @@ -596,25 +594,33 @@ // cache rebuild requested -> rebuild now $this->deleteDBCache($name . '_rebuild'); - return false; + if ( $this->rebuildDBCache($name, kCache::REBUILD_NOW, $max_rebuild_seconds, '[M1]') ) { + return false; + } } $cache = $cached_data[$name]; $rebuilding = $cached_data[$name . '_rebuilding']; if ( ($cache === false) && (!$rebuilding || $wait_seconds == 0) ) { // cache missing and nobody rebuilding it -> rebuild; enough waiting for cache to be ready + $this->rebuildDBCache($name, kCache::REBUILD_NOW, $max_rebuild_seconds, '[M2' . ($rebuilding ? 'R' : '!R') . ',WS=' . $wait_seconds . ']'); + return false; } elseif ( $cache !== false ) { // cache present -> return it + $this->cacheHandler->storeStatistics($name, $rebuilding ? 'h' : 'H'); + return $cache; } $wait_seconds -= kCache::WAIT_STEP; sleep(kCache::WAIT_STEP); } + $this->rebuildDBCache($name, kCache::REBUILD_NOW, $max_rebuild_seconds, '[M3' . ($rebuilding ? 'R' : '!R') . ',WS=' . $wait_seconds . ']'); + return false; } @@ -673,6 +679,8 @@ */ public function setDBCache($name, $value, $expiration = false) { + $this->cacheHandler->storeStatistics($name, 'WU'); + $this->deleteDBCache($name . '_rebuilding'); $this->_setDBCache($name, $value, $expiration); } @@ -683,11 +691,13 @@ * @param string $name * @param mixed $value * @param int|bool $expiration + * @param string $insert_type + * @return bool * @access protected */ - protected function _setDBCache($name, $value, $expiration = false) + protected function _setDBCache($name, $value, $expiration = false, $insert_type = 'REPLACE') { - if ((int)$expiration <= 0) { + if ( (int)$expiration <= 0 ) { $expiration = -1; } @@ -699,28 +709,58 @@ ); $this->Conn->nextQueryCachable = true; - $this->Conn->doInsert($fields_hash, TABLE_PREFIX . 'SystemCache', 'REPLACE'); + + return $this->Conn->doInsert($fields_hash, TABLE_PREFIX . 'SystemCache', $insert_type); } /** + * Sets value to database cache + * + * @param string $name + * @param mixed $value + * @param int|bool $expiration + * @return bool + * @access protected + */ + protected function _addDBCache($name, $value, $expiration = false) + { + return $this->_setDBCache($name, $value, $expiration, 'INSERT'); + } + + /** * Sets rebuilding mode for given cache * * @param string $name * @param int $mode * @param int $max_rebuilding_time - * @return void + * @param string $miss_type + * @return bool * @access public */ - public function rebuildDBCache($name, $mode = null, $max_rebuilding_time = 0) + public function rebuildDBCache($name, $mode = null, $max_rebuilding_time = 0, $miss_type = 'M') { if ( !isset($mode) || $mode == kCache::REBUILD_NOW ) { - $this->_setDBCache($name . '_rebuilding', 1, $max_rebuilding_time); + $this->cacheHandler->storeStatistics($name, $miss_type); + + if ( !$max_rebuilding_time ) { + return true; + } + + if ( !$this->_addDBCache($name . '_rebuilding', 1, $max_rebuilding_time) ) { + $this->cacheHandler->storeStatistics($name, 'l'); + + return false; + } + $this->deleteDBCache($name . '_rebuild'); + $this->cacheHandler->storeStatistics($name, 'L'); } elseif ( $mode == kCache::REBUILD_LATER ) { $this->_setDBCache($name . '_rebuild', 1, 0); $this->deleteDBCache($name . '_rebuilding'); } + + return true; } /** Index: branches/5.2.x/core/kernel/utility/unit_config_reader.php =================================================================== diff -u -N -r15130 -r15226 --- branches/5.2.x/core/kernel/utility/unit_config_reader.php (.../unit_config_reader.php) (revision 15130) +++ branches/5.2.x/core/kernel/utility/unit_config_reader.php (.../unit_config_reader.php) (revision 15226) @@ -1,6 +1,6 @@ Application->refreshModuleInfo(); if ( $this->Application->isCachingType(CACHING_TYPE_MEMORY) ) { - $data = $this->Application->getCache('master:config_files', false, CacheSettings::$unitCacheRebuildTime); + $data = $this->Application->getCache('master:config_files', false, $cache ? CacheSettings::$unitCacheRebuildTime : 0); } else { - $data = $this->Application->getDBCache('config_files', CacheSettings::$unitCacheRebuildTime); + $data = $this->Application->getDBCache('config_files', $cache ? CacheSettings::$unitCacheRebuildTime : 0); } if ( $data ) { @@ -197,15 +197,6 @@ } } else { - if ( $cache ) { - if ( $this->Application->isCachingType(CACHING_TYPE_MEMORY) ) { - $this->Application->rebuildCache('master:config_files', kCache::REBUILD_NOW, CacheSettings::$unitCacheRebuildTime); - } - else { - $this->Application->rebuildDBCache('config_files', kCache::REBUILD_NOW, CacheSettings::$unitCacheRebuildTime); - } - } - $this->findConfigFiles(FULL_PATH . DIRECTORY_SEPARATOR . 'core'); // search from core directory $this->findConfigFiles($folderPath); // search from modules directory Index: branches/5.2.x/core/kernel/utility/cache.php =================================================================== diff -u -N -r15096 -r15226 --- branches/5.2.x/core/kernel/utility/cache.php (.../cache.php) (revision 15096) +++ branches/5.2.x/core/kernel/utility/cache.php (.../cache.php) (revision 15226) @@ -1,6 +1,6 @@ _handler =& $handler; $this->cachingType = $handler->cachingType; $this->debugCache = $handler->cachingType == CACHING_TYPE_MEMORY && $this->Application->isDebugMode(); - $this->displayCacheStatistics = defined('DBG_CACHE') && DBG_CACHE && $this->Application->isDebugMode(); + $this->_storeStatistics = defined('DBG_CACHE') && DBG_CACHE; + + if ( $this->_storeStatistics ) { + // don't use FileHelper, since kFactory isn't ready yet + if ( !file_exists(RESTRICTED . DIRECTORY_SEPARATOR . 'cache_usage') ) { + mkdir(RESTRICTED . DIRECTORY_SEPARATOR . 'cache_usage'); + } + } } /** @@ -144,7 +145,6 @@ return $this->cachingType; } - /** * Stores value to cache * @@ -157,11 +157,16 @@ { // 1. stores current version of serial for given cache key $this->_setCache($name . '_serials', $this->replaceSerials($name), $expiration); + $this->storeStatistics($name, 'W'); - // 2. remove rebuilding mark + // 2. don't replace serials within the key + $saved = $this->_setCache($name, $value, $expiration); + $this->storeStatistics($name, 'U'); + + // 3. remove rebuilding mark $this->delete($name . '_rebuilding'); - return $this->_setCache($name, $value, $expiration); + return $saved; } /** @@ -181,22 +186,79 @@ } /** + * Stores value to cache (only if it's not there already) + * + * @param string $name + * @param mixed $value + * @param int $expiration cache record expiration time in seconds + * @return bool + */ + function addCache($name, $value, $expiration) + { + // 1. stores current version of serial for given cache key + $this->_setCache($name . '_serials', $this->replaceSerials($name), $expiration); + + // 2. remove rebuilding mark + $this->delete($name . '_rebuilding'); + + // 3. don't replace serials within the key + return $this->_addCache($name, $value, $expiration); + } + + /** + * Stores value to cache (only if it's not there already) + * + * @param string $name + * @param mixed $value + * @param int $expiration cache record expiration time in seconds + * @return bool + */ + function _addCache($name, $value, $expiration) + { + $prepared_name = $this->prepareKeyName($name); + $added = $this->_handler->add($prepared_name, $value, $expiration); + + if ( $added ) { + $this->_localStorage[$prepared_name] = $value; + } + + return $added; + } + + /** * Sets rebuilding mode for given cache * * @param string $name * @param int $mode * @param int $max_rebuilding_time + * @param string $miss_type + * @return bool */ - function rebuildCache($name, $mode = null, $max_rebuilding_time = 0) + function rebuildCache($name, $mode = null, $max_rebuilding_time = 0, $miss_type = 'M') { if ( !isset($mode) || $mode == self::REBUILD_NOW ) { - $this->_setCache($name . '_rebuilding', 1, $max_rebuilding_time); + $this->storeStatistics($name, $miss_type); + + if ( !$max_rebuilding_time ) { + return true; + } + + // prevent parallel rebuild attempt by using "add" instead of "set" method + if ( !$this->_addCache($name . '_rebuilding', 1, $max_rebuilding_time) ) { + $this->storeStatistics($name, 'l'); + + return false; + } + + $this->storeStatistics($name, 'L'); $this->delete($name . '_rebuild'); } elseif ( $mode == self::REBUILD_LATER ) { $this->_setCache($name . '_rebuild', 1, 0); $this->delete($name . '_rebuilding'); } + + return true; } /** @@ -209,15 +271,23 @@ */ function getCache($name, $store_locally = true, $max_rebuild_seconds = 0) { - $cached_data = $this->_getCache(Array ($name . '_rebuild', $name . '_serials'), Array (true, true)); + $cached_data = $this->_getCache(Array ($name . '_rebuild', $name . '_serials'), Array (true, false)); if ( $cached_data[$name . '_rebuild'] ) { // cache rebuild requested -> rebuild now $this->delete($name . '_rebuild'); - return false; + if ( $this->rebuildCache($name, self::REBUILD_NOW, $max_rebuild_seconds, '[M1]') ) { + return false; + } } + // There are 2 key types: + // - with serials, e.g. with_serial_key[%LangSerial%] + // - without serials, e.g. without_serial + // Evaluated serials of each cache key are stored in '{$name}_serials' cache key. + // If cache is present, but serial is outdated, then cache value is assumed to be outdated. + $new_serial = $this->replaceSerials($name); $old_serial = $cached_data[$name . '_serials']; @@ -232,21 +302,41 @@ if ( ($cache === false) && (!$rebuilding || $wait_seconds == 0) ) { // cache missing and nobody rebuilding it -> rebuild; enough waiting for cache to be ready + $this->rebuildCache($name, self::REBUILD_NOW, $max_rebuild_seconds, '[M2' . ($rebuilding ? 'R' : '!R') . ',WS=' . $wait_seconds . ']'); + return false; } elseif ( $cache !== false ) { + // re-read serial, since it might have been changed in parallel process !!! + $old_serial = $this->_getCache($name . '_serials', false); + // cache present (if other user is rebuilding it, then it's outdated cache) -> return it - return $rebuilding || $new_serial == $old_serial ? $cache : false; + if ( $rebuilding || $new_serial == $old_serial ) { + $this->storeStatistics($name, $rebuilding ? 'h' : 'H'); + + return $cache; + } + + $this->rebuildCache($name, self::REBUILD_NOW, $max_rebuild_seconds, '[M3' . ($rebuilding ? 'R' : '!R') . ',WS=' . $wait_seconds . ']'); + + return false; } $wait_seconds -= self::WAIT_STEP; sleep(self::WAIT_STEP); } + } - return $cache; + $cache = $this->_getCache($name, $store_locally); + + if ( $cache === false ) { + $this->rebuildCache($name, self::REBUILD_NOW, $max_rebuild_seconds, '[M4]'); } + else { + $this->storeStatistics($name, 'H'); + } - return $this->_getCache($name, $store_locally); + return $cache; } /** @@ -269,10 +359,6 @@ $name = $names[$index]; if ( $store_locally[$index] && array_key_exists($prepared_name, $this->_localStorage) ) { - if ( $this->displayCacheStatistics ) { - $this->setStatistics($prepared_name, $this->_localStorage[$prepared_name]); - } - $res[$name] = $this->_localStorage[$prepared_name]; unset($to_get[$index]); } @@ -330,10 +416,6 @@ if ( $store_locally /*&& ($res !== false)*/ ) { $this->_localStorage[$name] = $res; - - if ( $this->displayCacheStatistics ) { - $this->setStatistics($name, $res); - } } } @@ -441,58 +523,24 @@ return "{$this->siteKeyName}:{$this->siteKeyValue}:"; } - function setStatistics($name, $found) - { - if (strpos($name, ']:') !== false) { - list ($cache_name, $name) = explode(']:', $name, 2); - } - else { - $cache_name = '-'; - } - - if (!array_key_exists($cache_name, $this->statistics)) { - $this->statistics[$cache_name] = Array (); - } - - if (!array_key_exists($name, $this->statistics[$cache_name])) { - $this->statistics[$cache_name][$name] = Array (); - } - - $status_key = $found ? 'found' : 'not_found'; - - if (!isset($this->statistics[$cache_name][$name][$status_key])) { - $this->statistics[$cache_name][$name][$status_key] = 0; - } - - $this->statistics[$cache_name][$name][$status_key]++; - } - /** - * Returns storage size in bytes + * Stores statistics about cache usage in a file (one file per cache) * - * @return int + * @param string $name + * @param string $action_type {M - miss, L - lock, W - write, U - unlock, H - actual hit, h - outdated hit} + * @return void + * @access public */ - function getStorageSize() + public function storeStatistics($name, $action_type) { - return strlen( serialize($this->_localStorage) ); - } - - function printStatistics() - { - $cache_size = $this->getStorageSize(); - - $this->Application->Debugger->appendHTML('Cache Size: ' . kUtil::formatSize($cache_size) . ' (' . $cache_size . ')'); - - foreach ($this->statistics as $cache_name => $cache_data) { - foreach ($cache_data as $key => $value) { - if (!array_key_exists('found', $value) || $value['found'] == 1) { - // remove cached records, that were used only 1 or 2 times - unset($this->statistics[$cache_name][$key]); - } - } + if ( !$this->_storeStatistics ) { + return; } - kUtil::print_r($this->statistics, 'Cache Statistics:'); + $name = str_replace(Array ('/', '\\', ':'), '_', $name); + $fp = fopen(RESTRICTED . DIRECTORY_SEPARATOR . 'cache_usage' . DIRECTORY_SEPARATOR . $name, 'a'); + fwrite($fp, $action_type); + fclose($fp); } } @@ -541,6 +589,19 @@ } /** + * Stores value in cache (only if it's not there already) + * + * @param string $name + * @param mixed $value + * @param int $expiration + * @return bool + */ + function add($name, $value, $expiration = 0) + { + return true; + } + + /** * Deletes key from cach * * @param string $name @@ -632,6 +693,20 @@ } /** + * Stores value in cache (only if it's not there already) + * + * @param string $name + * @param mixed $value + * @param int $expiration + * @return bool + */ + function add($name, $value, $expiration = 0) + { + // 0 - don't use compression + return $this->_handler->add($name, $value, 0, $expiration); + } + + /** * Deletes key from cache * * @param string $name @@ -695,6 +770,19 @@ } /** + * Stores value in cache (only if it's not there already) + * + * @param string $name + * @param mixed $value + * @param int $expiration + * @return bool + */ + function add($name, $value, $expiration = 0) + { + return apc_add($name, $value, $expiration); + } + + /** * Deletes key from cache * * @param string $name @@ -767,6 +855,24 @@ } /** + * Stores value in cache (only if it's not there already) + * + * @param string $name + * @param mixed $value + * @param int $expiration + * @return bool + */ + function add($name, $value, $expiration = 0) + { + // not atomic operation, like in Memcached and may fail + if ( xcache_isset($name) ) { + return false; + } + + return $this->set($name, $value, $expiration); + } + + /** * Deletes key from cache * * @param string $name Index: branches/5.2.x/core/units/helpers/site_helper.php =================================================================== diff -u -N -r14976 -r15226 --- branches/5.2.x/core/units/helpers/site_helper.php (.../site_helper.php) (revision 14976) +++ branches/5.2.x/core/units/helpers/site_helper.php (.../site_helper.php) (revision 15226) @@ -1,6 +1,6 @@ Application->isCachingType(CACHING_TYPE_MEMORY) ) { - $this->Application->rebuildCache('master:domains_parsed', kCache::REBUILD_NOW, CacheSettings::$domainsParsedRebuildTime); - } - else { - $this->Application->rebuildDBCache('domains_parsed', kCache::REBUILD_NOW, CacheSettings::$domainsParsedRebuildTime); - } - $sql = 'SELECT * FROM ' . TABLE_PREFIX . 'SiteDomains ORDER BY Priority DESC'; Index: branches/5.2.x/core/units/helpers/menu_helper.php =================================================================== diff -u -N -r15137 -r15226 --- branches/5.2.x/core/units/helpers/menu_helper.php (.../menu_helper.php) (revision 15137) +++ branches/5.2.x/core/units/helpers/menu_helper.php (.../menu_helper.php) (revision 15226) @@ -1,6 +1,6 @@ parentPaths = $menu['parentPaths']; } else { - if ( $this->Application->isCachingType(CACHING_TYPE_MEMORY) ) { - $this->Application->rebuildCache('master:cms_menu', kCache::REBUILD_NOW, CacheSettings::$cmsMenuRebuildTime); - } - else { - $this->Application->rebuildDBCache('cms_menu', kCache::REBUILD_NOW, CacheSettings::$cmsMenuRebuildTime); - } - $menu = $this->_altBuildMenuStructure(Array ('CategoryId' => $root_cat, 'ParentPath' => $root_path)); $menu['parentPaths'] = $this->parentPaths;