Project

General

Profile

« Previous | Next » 

Revision 761

Added by thorn about 16 years ago

search: great speed-up with large pages - requires PHP >= 4.3.3; small speed-up for PHP < 4.3.3.
FCK-Editor: loads large pages faster

View differences:

search_modext.php
30 30 
		$str = implode(" ", $search_url_array);
31 31 
		$link = "?searchresult=1&amp;sstring=".urlencode($str);
32 32 
	} else {
33 
		$str = strtr($search_url_array[0], " ", "_");
33 
		$str = str_replace(' ', '_', $search_url_array[0]);
34 34 
		$link = "?searchresult=2&amp;sstring=".urlencode($str);
35 35 
	}
36 36 
	return $link;
......
77 77 
// checks if _any_ of the searchwords matches
78 78 
function is_any_matched($text, $search_words) {
79 79 
	$any_matched = false;
80 
	$word = "(".implode("|", $search_words).")";
80 
	$word = '('.implode('|', $search_words).')';
81 81 
	if(preg_match('/'.$word.'/iu', $text)) {
82 82 
		$any_matched = true;
83 83 
	}
......
88 88 
function get_excerpts($text, $search_words, $max_excerpt_num) {
89 89 
	$match_array = array();
90 90 
	$excerpt_array = array();
91 
	$word = "(".implode("|", $search_words).")";
92
93 
	$text = strtr($text, array('&lt;'=>'<', '&gt;'=>'>', '&amp;'=>'&', '&quot;'=>'"', '&#39;'=>'\'', '&nbsp;'=>"\xC2\xA0"));
94 
	$word = strtr($word, array('&lt;'=>'<', '&gt;'=>'>', '&amp;'=>'&', '&quot;'=>'"', '&#39;'=>'\'', '&nbsp;'=>"\xC2\xA0"));
91 
	$word = '('.implode('|', $search_words).')';
95 92 
	// Build the regex-string
96 
	//...INVERTED EXCLAMATION MARK - INVERTED QUESTION MARK - DOUBLE EXCLAMATION MARK - INTERROBANG - EXCLAMATION QUESTION MARK - QUESTION EXCLAMATION MARK - DOUBLE QUESTION MARK - HALFWIDTH IDEOGRAPHIC FULL STOP - IDEOGRAPHIC FULL STOP - IDEOGRAPHIC COMMA
93 
	// start-sign: .!?; + INVERTED EXCLAMATION MARK - INVERTED QUESTION MARK - DOUBLE EXCLAMATION MARK - INTERROBANG - EXCLAMATION QUESTION MARK - QUESTION EXCLAMATION MARK - DOUBLE QUESTION MARK - HALFWIDTH IDEOGRAPHIC FULL STOP - IDEOGRAPHIC FULL STOP - IDEOGRAPHIC COMMA
97 94 
	$str1=".!?;"."\xC2\xA1"."\xC2\xBF"."\xE2\x80\xBC"."\xE2\x80\xBD"."\xE2\x81\x89"."\xE2\x81\x88"."\xE2\x81\x87"."\xEF\xBD\xA1"."\xE3\x80\x82"."\xE3\x80\x81";
98 
	// ...DOUBLE EXCLAMATION MARK - INTERROBANG - EXCLAMATION QUESTION MARK - QUESTION EXCLAMATION MARK - DOUBLE QUESTION MARK - HALFWIDTH IDEOGRAPHIC FULL STOP - IDEOGRAPHIC FULL STOP - IDEOGRAPHIC COMMA
95 
	// stop-sign: .!?; + DOUBLE EXCLAMATION MARK - INTERROBANG - EXCLAMATION QUESTION MARK - QUESTION EXCLAMATION MARK - DOUBLE QUESTION MARK - HALFWIDTH IDEOGRAPHIC FULL STOP - IDEOGRAPHIC FULL STOP - IDEOGRAPHIC COMMA
99 96 
	$str2=".!?;"."\xE2\x80\xBC"."\xE2\x80\xBD"."\xE2\x81\x89"."\xE2\x81\x88"."\xE2\x81\x87"."\xEF\xBD\xA1"."\xE3\x80\x82"."\xE3\x80\x81";
100 
	// "{0,200}?'.$word.'" : the '?' (ungreedy) is for performance-tuning; try a step-by-step-trace to see what i mean.
101 
	if(preg_match_all('/(?:^|\b|['.$str1.'])([^'.$str1.']{0,200}?'.$word.'[^'.$str2.']{0,200}(?:['.$str2.']|\b|$))/isu', $text, $match_array)) {
102 
		foreach($match_array[1] AS $string) {
103 
			$excerpt_array[] = trim($string);
97 
	$regex='/(?:^|\b|['.$str1.'])([^'.$str1.']{0,200}?'.$word.'[^'.$str2.']{0,200}(?:['.$str2.']|\b|$))/Sisu';
98 
	if(version_compare(PHP_VERSION, '4.3.3', '>=') == 1) {
99 
		// jump from match to match, get excerpt, stop if $max_excerpt_num is reached
100 
		$last_end = 0; $offset = 0;
101 
		while(preg_match('/'.$word.'/Sisu', $text, $match_array, PREG_OFFSET_CAPTURE, $last_end)) {
102 
			$offset = ($match_array[0][1]-206 < $last_end)?$last_end:$match_array[0][1]-206;
103 
			if(preg_match($regex, $text, $matches, PREG_OFFSET_CAPTURE, $offset)) {
104 
				$last_end = $matches[1][1]+strlen($matches[1][0])-1;
105 
				if(!preg_match('/\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\./S', $matches[1][0])) // skip excerpts with email-addresses
106 
					$excerpt_array[] = trim($matches[1][0]);
107 
				if(count($excerpt_array)>=$max_excerpt_num) {
108 
					$excerpt_array = array_unique($excerpt_array);
109 
					if(count($excerpt_array) >= $max_excerpt_num)
110 
						break;
111 
				}
112 
			} else { // problem - preg_match failed: can't find a start- or stop-sign
113 
				$last_end += 201; // jump forward and try again
114 
			}
104 115 
		}
116 
	} else { // compatile, but may be very slow with many large pages
117 
		if(preg_match_all($regex, $text, $match_array)) {
118 
			foreach($match_array[1] AS $string) {
119 
				if(!preg_match('/\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\./S', $string)) // skip excerpts with email-addresses
120 
					$excerpt_array[] = trim($string);
121 
			}
122 
		}
105 123 
	}
106 124 
	return $excerpt_array;
107 125 
}
......
121 139 
	}
122 140 
	// prepare search-string
123 141 
	$string = "(".implode("|", $search_words).")";
124 
	$string = strtr($string, array('&lt;'=>'<', '&gt;'=>'>', '&amp;'=>'&', '&quot;'=>'"', '&#39;'=>'\'', '&nbsp;'=>"\xC2\xA0"));
125 142 
	// we want markup on search-results page,
126 143 
	// but we need some 'magic' to prevent <br />, <b>... from being highlighted
127 144 
	$excerpt = '';
128 145 
	foreach($excerpt_array as $str) {
129 146 
		$excerpt .= '#,,#'.preg_replace("/($string)/iu","#,,,,#$1#,,,,,#",$str).'#,,,#';
130 147 
	}
131 
	$excerpt = strtr($excerpt, array('<'=>'&lt;', '>'=>'&gt;', '&'=>'&amp;', '"'=>'&quot;', '\''=>'&#39;'));
132 
	$excerpt = strtr($excerpt, array('#,,,,#'=>$EXCERPT_MARKUP_START, '#,,,,,#'=>$EXCERPT_MARKUP_END));
133 
	$excerpt = strtr($excerpt, array('#,,#'=>$EXCERPT_BEFORE, '#,,,#'=>$EXCERPT_AFTER));
148 
	$excerpt = str_replace(array('&','<','>','"','\'',"\xC2\xA0"), array('&amp;','&lt;','&gt;','&quot;','&#39;','&nbsp;'), $excerpt);
149 
	$excerpt = str_replace(array('#,,,,#','#,,,,,#'), array($EXCERPT_MARKUP_START,$EXCERPT_MARKUP_END), $excerpt);
150 
	$excerpt = str_replace(array('#,,#','#,,,#'), array($EXCERPT_BEFORE,$EXCERPT_AFTER), $excerpt);
134 151 
	// prepare to write out
135 152 
	if(DEFAULT_CHARSET != 'utf-8') {
136 153 
		$excerpt = umlauts_to_entities($excerpt, 'UTF-8');
......
138 155 
	return $excerpt;
139 156 
}
140 157 

  		




  141
  
  
    
// work out what the link-target should be


  		




  
  158
  
    
// work out what the link-anchor should be


  		




  142
  159
  
    
function make_url_target($page_link_target, $text, $search_words) {


  		




  143
  160
  
    
	// 1. e.g. $page_link_target=="&monthno=5&year=2007" - module-dependent target. Do nothing.


  		




  144
  161
  
    
	// 2. $page_link_target=="#!wb_section_..." - the user wants the section-target, so do nothing.


  		




  145
  162
  
    
	// 3. $page_link_target=="#wb_section_..." - try to find a better target, use the section-target as fallback.


  		




  146
  163
  
    
	// 4. $page_link_target=="" - do nothing


  		




  147
  
  
    
	if(version_compare(PHP_VERSION, '4.3.0', ">=") && substr($page_link_target,0,12)=='#wb_section_') {


  		




  
  164
  
    
	if(version_compare(PHP_VERSION, '4.3.3', ">=") && substr($page_link_target,0,12)=='#wb_section_') {


  		




  148
  165
  
    
		$word = '('.implode('|', $search_words).')';


  		




  149
  166
  
    
		preg_match('/'.$word.'/iu', $text, $match, PREG_OFFSET_CAPTURE);


  		




  150
  167
  
    
		if($match && is_array($match[0])) {


  		




  151
  
  
    
			$x=$match[0][1];


  		




  
  168
  
    
			$x=$match[0][1]; // position of first match


  		




  152
  169
  
    
			// is there an anchor nearby?


  		




  153
  
  
    
			if (preg_match_all('/<(?:[^>]+id|\s*a[^>]+name)\s*=\s*"(.*)"/iuU', $text, $match, PREG_OFFSET_CAPTURE)) {


  		




  
  170
  
    
			if(preg_match_all('/<(?:[^>]+id|\s*a[^>]+name)\s*=\s*"(.*)"/SiU', substr($text,0,$x), $match, PREG_OFFSET_CAPTURE)) {


  		




  154
  171
  
    
				$anchor='';


  		




  155
  172
  
    
				foreach($match[1] AS $array) {


  		




  156
  173
  
    
					if($array[1] > $x) {


  		




  ......




  210
  227
  
    
	if(!isset($mod_max_excerpt_num))    $mod_max_excerpt_num = $func_default_max_excerpt;


  		




  211
  228
  
    
	if(!isset($mod_pic_link))           $mod_pic_link = "";


  		




  212
  229
  
    
	if(!isset($mod_no_highlight))       $mod_no_highlight = false;


  		




  
  230
  
    
	if(!isset($func_enable_flush))      $func_enable_flush = false; // set this in db: wb_search.cfg_enable_flush [READ THE DOC BEFORE]


  		




  213
  231
  
    
	if($mod_text == "") // nothing to do


  		




  214
  232
  
    
		{ return false; }


  		




  215
  233
  
    
	if($mod_no_highlight) // no highlighting


  		




  216
  234
  
    
		{ $mod_page_link_target = "&amp;nohighlight=1".$mod_page_link_target; }


  		




  217
  235
  
    

  		




  218
  
  
    
	// prepare the text (part 1): remove lf and cr, convert \" to ", remove comments, style, scripting and unnecessary whitespace, convert to utf8


  		




  219
  
  
    
	$mod_text = strtr($mod_text, array("\x0D\x0A" => ' ', "\x0D" => ' ', "\x0A" => ' ', '\"' => '"'));


  		




  220
  
  
    
	$mod_text = preg_replace('/(<!--.*?-->|<style.*?<\/style>|<script.*?<\/script>|\s+)/i', ' ', $mod_text);


  		




  221
  
  
    
	// remove email-addresses


  		




  222
  
  
    
	$mod_text = preg_replace('/\b[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,4}\b/', ' ', $mod_text);


  		




  
  236
  
    
	// clean the text:


  		




  
  237
  
    
	$mod_text = str_replace(array("\x0D","\x0A"), ' ', $mod_text);


  		




  
  238
  
    
	$mod_text = preg_replace('#<(!--.*--|style.*</style|script.*</script)>#SiU', ' ', $mod_text);


  		




  
  239
  
    
	$mod_text = preg_replace('#<(br( /)?|dt|/dd|/?(h[1-6]|tr|table|p|li|ul|pre|code|div|hr))[^>]*>#Si', '.', $mod_text);


  		




  223
  240
  
    
	$mod_text = entities_to_umlauts($mod_text, 'UTF-8');


  		




  224
  
  
    

  		




  225
  
  
    
	// make the link from $mod_page_link, add target


  		




  226
  
  
    
	$link = "";


  		




  227
  
  
    
	$link = page_link($mod_page_link);


  		




  228
  
  
    
	$link .= make_url_searchstring($func_search_match, $func_search_url_array);


  		




  229
  
  
    
	$link .= make_url_target($mod_page_link_target, $mod_text, $func_search_words);


  		




  230
  
  
    

  		




  231
  
  
    
	// prepare the text (part 2): convert some special tags to '.', strip tags


  		




  232
  
  
    
	$mod_text = preg_replace('/<(br( \/)?|dt|\/dd|\/?(h[1-6]|tr|table|p|li|ul|pre|code|div|hr))[^>]*>/i', '.', $mod_text);


  		




  
  241
  
    
	// search for an better anchor - this have to be done before strip_tags() (may fail if search-string contains <, &, amp, gt, lt, ...)


  		




  
  242
  
    
	$anchor =  make_url_target($mod_page_link_target, $mod_text, $func_search_words);


  		




  233
  243
  
    
	$mod_text = strip_tags($mod_text);


  		




  234
  
  
    

  		




  
  244
  
    
	$mod_text = str_replace(array('&gt;','&lt;','&amp;','&quot;','&#39;','&apos;','&nbsp;'), array('>','<','&','"','\'','\'',"\xC2\xA0"), $mod_text);


  		




  235
  245
  
    
	// Do a fast scan over $mod_text first. This will speedup things a lot.


  		




  236
  246
  
    
	if($func_search_match == 'all') {


  		




  237
  
  
    
		if(!is_all_matched($mod_text, $func_search_words)) {


  		




  
  247
  
    
		if(!is_all_matched($mod_text, $func_search_words))


  		




  238
  248
  
    
			return false;


  		




  239
  
  
    
		}


  		




  240
  249
  
    
	}


  		




  241
  250
  
    
	elseif(!is_any_matched($mod_text, $func_search_words)) {


  		




  242
  251
  
    
		return false;


  		




  243
  252
  
    
	}


  		




  244
  253
  
    

  		




  
  254
  
    
	// make the link from $mod_page_link, add anchor


  		




  
  255
  
    
	$link = "";


  		




  
  256
  
    
	$link = page_link($mod_page_link);


  		




  
  257
  
    
	$link .= make_url_searchstring($func_search_match, $func_search_url_array);


  		




  
  258
  
    
	$link .= $anchor;


  		




  
  259
  
    

  		




  245
  260
  
    
	// now get the excerpt


  		




  246
  261
  
    
	$excerpt = "";


  		




  247
  262
  
    
	$excerpt_array = array();


  		




  ......




  252
  267
  
    
		$excerpt = prepare_excerpts($excerpt_array, $func_search_words, $mod_max_excerpt_num);


  		




  253
  268
  
    
	}


  		




  254
  269
  
    

  		




  255
  
  
    
	// handle image


  		




  
  270
  
    
	// handle thumbs - to deactivate this look in the module's search.php: $show_thumb (or maybe in the module's settings-page)


  		




  256
  271
  
    
	if($mod_pic_link != "") {


  		




  257
  
  
    
		$excerpt = '<table width="100%" cellspacing="0" cellpadding="0" border="0"><tbody><tr><td width="110" valign="top"><a href="'.$link.'"><img src="'.WB_URL.'/'.MEDIA_DIRECTORY.$mod_pic_link.'" alt="" /></a></td><td>'.$excerpt.'</td></tr></tbody></table>';


  		




  
  272
  
    
		$excerpt = '<table class="excerpt_thumb" width="100%" cellspacing="0" cellpadding="0" border="0"><tbody><tr><td width="110" valign="top"><a href="'.$link.'"><img src="'.WB_URL.'/'.MEDIA_DIRECTORY.$mod_pic_link.'" alt="" /></a></td><td>'.$excerpt.'</td></tr></tbody></table>';


  		




  258
  273
  
    
	}


  		




  259
  274
  
    

  		




  260
  275
  
    
	// print-out the excerpt


  		




  ......




  276
  291
  
    
		$excerpt


  		




  277
  292
  
    
	);


  		




  278
  293
  
    
	echo str_replace($vars, $values, $func_results_loop_string);


  		




  
  294
  
    
	if($func_enable_flush) { // ATTN: this will bypass output-filters and may break template-layout or -filters


  		




  
  295
  
    
		ob_flush();flush();


  		




  
  296
  
    
	}


  		




  279
  297
  
    
	return true;


  		




  280
  298
  
    
}


  		




  281
  299
  
    

  		




  ......




  306
  324
  
    

  		




  307
  325
  
    
// keeps only wanted entries in array $files. $str have to be an eregi()-compatible regex


  		




  308
  326
  
    
function clear_filelist($files, $str, $keep=true) {


  		




  309
  
  
    
	// $keep = true  : remove all non-matching entries


  		




  310
  
  
    
	// $keep = false : remove all matching entries


  		




  
  327
  
    
	// options: $keep = true  : remove all non-matching entries


  		




  
  328
  
    
	//          $keep = false : remove all matching entries


  		




  311
  329
  
    
	$c_filelist = array();


  		




  312
  330
  
    
	if($str == '')


  		




  313
  331
  
    
		return $files;


  		












Also available in:  Unified diff