Project

General

Profile

« Previous | Next » 

Revision 761

Added by thorn over 16 years ago

search: great speed-up with large pages - requires PHP >= 4.3.3; small speed-up for PHP < 4.3.3.
FCK-Editor: loads large pages faster

View differences:

functions-utf8.php
24 24
*/
25 25

  
26 26
/*
27
 * A large part of this file is based on 'utf8.php' from the DokuWiki-project.
27
 * A part of this file is based on 'utf8.php' from the DokuWiki-project.
28 28
 * (http://www.splitbrain.org/projects/dokuwiki):
29 29
 **
30 30
 * UTF8 helper functions
......
33 33
 **
34 34
 * modified for use with Website Baker
35 35
 * from thorn, Jan. 2008
36
 *
37
 * most of the original functions appeared to be to slow with large strings, so i replaced them with my own ones
38
 * thorn, Mar. 2008
36 39
 */
37 40

  
38 41
// Functions we use in Website Baker:
......
48 51
/*
49 52
 * check for mb_string support
50 53
 */
54
//define('UTF8_NOMBSTRING',1); // uncomment this to forbid use of mb_string-functions
51 55
if(!defined('UTF8_MBSTRING')){
52 56
  if(function_exists('mb_substr') && !defined('UTF8_NOMBSTRING')){
53 57
    define('UTF8_MBSTRING',1);
......
73 77
}
74 78

  
75 79
/*
76
 * Strips all highbyte chars
77
 *
78
 * Returns a pure ASCII7 string
79
 *
80
 * @author Andreas Gohr <andi@splitbrain.org>
81
 */
82
function utf8_strip($str){
83
  $ascii = '';
84
  for($i=0; $i<strlen($str); $i++){
85
    if(ord($str{$i}) <128){
86
      $ascii .= $str{$i};
87
    }
88
  }
89
  return $ascii;
90
}
91

  
92
/*
93 80
 * Tries to detect if a string is in Unicode encoding
94 81
 *
95 82
 * @author <bmorel@ssi.fr>
......
114 101
}
115 102

  
116 103
/*
117
 * Unicode aware replacement for strlen()
118
 *
119
 * utf8_decode() converts characters that are not in ISO-8859-1
120
 * to '?', which, for the purpose of counting, is alright - It's
121
 * even faster than mb_strlen.
122
 *
123
 * @author <chernyshevsky at hotmail dot com>
124
 * @see    strlen()
125
 * @see    utf8_decode()
126
 */
127
function utf8_strlen($string){
128
  return strlen(utf8_decode($string));
129
}
130

  
131
/*
132
 * UTF-8 aware alternative to substr
133
 *
134
 * Return part of a string given character offset (and optionally length)
135
 *
136
 * @author Harry Fuecks <hfuecks@gmail.com>
137
 * @author Chris Smith <chris@jalakai.co.uk>
138
 * @param string
139
 * @param integer number of UTF-8 characters offset (from left)
140
 * @param integer (optional) length in UTF-8 characters from offset
141
 * @return mixed string or false if failure
142
 */
143
function utf8_substr($str, $offset, $length = null) {
144
    if(UTF8_MBSTRING){
145
        if( $length === null ){
146
            return mb_substr($str, $offset);
147
        }else{
148
            return mb_substr($str, $offset, $length);
149
        }
150
    }
151

  
152
    /*
153
     * Notes:
154
     *
155
     * no mb string support, so we'll use pcre regex's with 'u' flag
156
     * pcre only supports repetitions of less than 65536, in order to accept up to MAXINT values for
157
     * offset and length, we'll repeat a group of 65535 characters when needed (ok, up to MAXINT-65536)
158
     *
159
     * substr documentation states false can be returned in some cases (e.g. offset > string length)
160
     * mb_substr never returns false, it will return an empty string instead.
161
     *
162
     * calculating the number of characters in the string is a relatively expensive operation, so
163
     * we only carry it out when necessary. It isn't necessary for +ve offsets and no specified length
164
     */
165

  
166
    // cast parameters to appropriate types to avoid multiple notices/warnings
167
    $str = (string)$str;                          // generates E_NOTICE for PHP4 objects, but not PHP5 objects
168
    $offset = (int)$offset;
169
    if (!is_null($length)) $length = (int)$length;
170

  
171
    // handle trivial cases
172
    if ($length === 0) return '';
173
    if ($offset < 0 && $length < 0 && $length < $offset) return '';
174

  
175
    $offset_pattern = '';
176
    $length_pattern = '';
177

  
178
    // normalise -ve offsets (we could use a tail anchored pattern, but they are horribly slow!)
179
    if ($offset < 0) {
180
      $strlen = strlen(utf8_decode($str));        // see notes
181
      $offset = $strlen + $offset;
182
      if ($offset < 0) $offset = 0;
183
    }
184

  
185
    // establish a pattern for offset, a non-captured group equal in length to offset
186
    if ($offset > 0) {
187
      $Ox = (int)($offset/65535);
188
      $Oy = $offset%65535;
189

  
190
      if ($Ox) $offset_pattern = '(?:.{65535}){'.$Ox.'}';
191
      $offset_pattern = '^(?:'.$offset_pattern.'.{'.$Oy.'})';
192
    } else {
193
      $offset_pattern = '^';                      // offset == 0; just anchor the pattern
194
    }
195

  
196
    // establish a pattern for length
197
    if (is_null($length)) {
198
      $length_pattern = '(.*)$';                  // the rest of the string
199
    } else {
200

  
201
      if (!isset($strlen)) $strlen = strlen(utf8_decode($str));    // see notes
202
      if ($offset > $strlen) return '';           // another trivial case
203

  
204
      if ($length > 0) {
205

  
206
        $length = min($strlen-$offset, $length);  // reduce any length that would go passed the end of the string
207

  
208
        $Lx = (int)($length/65535);
209
        $Ly = $length%65535;
210

  
211
        // +ve length requires ... a captured group of length characters
212
        if ($Lx) $length_pattern = '(?:.{65535}){'.$Lx.'}';
213
        $length_pattern = '('.$length_pattern.'.{'.$Ly.'})';
214

  
215
      } else if ($length < 0) {
216

  
217
        if ($length < ($offset - $strlen)) return '';
218

  
219
        $Lx = (int)((-$length)/65535);
220
        $Ly = (-$length)%65535;
221

  
222
        // -ve length requires ... capture everything except a group of -length characters
223
        //                         anchored at the tail-end of the string
224
        if ($Lx) $length_pattern = '(?:.{65535}){'.$Lx.'}';
225
        $length_pattern = '(.*)(?:'.$length_pattern.'.{'.$Ly.'})$';
226
      }
227
    }
228

  
229
    if (!preg_match('#'.$offset_pattern.$length_pattern.'#us',$str,$match)) return '';
230
    return $match[1];
231
}
232

  
233
/*
234
 * Unicode aware replacement for substr_replace()
235
 *
236
 * @author Andreas Gohr <andi@splitbrain.org>
237
 * @see    substr_replace()
238
 */
239
function utf8_substr_replace($string, $replacement, $start , $length=0 ){
240
  $ret = '';
241
  if($start>0) $ret .= utf8_substr($string, 0, $start);
242
  $ret .= $replacement;
243
  $ret .= utf8_substr($string, $start+$length);
244
  return $ret;
245
}
246

  
247
/*
248
 * Unicode aware replacement for ltrim()
249
 *
250
 * @author Andreas Gohr <andi@splitbrain.org>
251
 * @see    ltrim()
252
 * @return string
253
 */
254
function utf8_ltrim($str,$charlist=''){
255
  if($charlist == '') return ltrim($str);
256

  
257
  //quote charlist for use in a characterclass
258
  $charlist = preg_replace('!([\\\\\\-\\]\\[/])!','\\\${1}',$charlist);
259

  
260
  return preg_replace('/^['.$charlist.']+/u','',$str);
261
}
262

  
263
/*
264
 * Unicode aware replacement for rtrim()
265
 *
266
 * @author Andreas Gohr <andi@splitbrain.org>
267
 * @see    rtrim()
268
 * @return string
269
 */
270
function  utf8_rtrim($str,$charlist=''){
271
  if($charlist == '') return rtrim($str);
272

  
273
  //quote charlist for use in a characterclass
274
  $charlist = preg_replace('!([\\\\\\-\\]\\[/])!','\\\${1}',$charlist);
275

  
276
  return preg_replace('/['.$charlist.']+$/u','',$str);
277
}
278

  
279
/*
280
 * Unicode aware replacement for trim()
281
 *
282
 * @author Andreas Gohr <andi@splitbrain.org>
283
 * @see    trim()
284
 * @return string
285
 */
286
function  utf8_trim($str,$charlist='') {
287
  if($charlist == '') return trim($str);
288

  
289
  return utf8_ltrim(utf8_rtrim($str,$charlist),$charlist);
290
}
291

  
292
/*
293
 * This is a unicode aware replacement for strtolower()
294
 *
295
 * Uses mb_string extension if available
296
 *
297
 * @author Leo Feyer <leo@typolight.org>
298
 * @see    strtolower()
299
 * @see    utf8_strtoupper()
300
 */
301
function utf8_strtolower($string){
302
  if(UTF8_MBSTRING) return mb_strtolower($string,'utf-8');
303

  
304
  global $UTF8_UPPER_TO_LOWER;
305
  return strtr($string,$UTF8_UPPER_TO_LOWER);
306
}
307

  
308
/*
309
 * This is a unicode aware replacement for strtoupper()
310
 *
311
 * Uses mb_string extension if available
312
 *
313
 * @author Leo Feyer <leo@typolight.org>
314
 * @see    strtoupper()
315
 * @see    utf8_strtoupper()
316
 */
317
function utf8_strtoupper($string){
318
  if(UTF8_MBSTRING) return mb_strtoupper($string,'utf-8');
319

  
320
  global $UTF8_LOWER_TO_UPPER;
321
  return strtr($string,$UTF8_LOWER_TO_UPPER);
322
}
323

  
324
/*
325 104
 * Romanize a non-latin string
326 105
 *
327 106
 * @author Andreas Gohr <andi@splitbrain.org>
......
356 135
}
357 136

  
358 137
/*
359
 * This is an Unicode aware replacement for strpos
360
 *
361
 * @author Leo Feyer <leo@typolight.org>
362
 * @see    strpos()
363
 * @param  string
364
 * @param  string
365
 * @param  integer
366
 * @return integer
138
 * added functions - thorn
367 139
 */
368
function utf8_strpos($haystack, $needle, $offset=0){
369
    $comp = 0;
370
    $length = null;
371 140

  
372
    while (is_null($length) || $length < $offset) {
373
        $pos = strpos($haystack, $needle, $offset + $comp);
374

  
375
        if ($pos === false)
376
            return false;
377

  
378
        $length = utf8_strlen(substr($haystack, 0, $pos));
379

  
380
        if ($length < $offset)
381
            $comp = $pos - $length;
382
    }
383

  
384
    return $length;
385
}
386

  
387 141
/*
388
 * Encodes UTF-8 characters to HTML entities
389
 *
390
 * @author Tom N Harris <tnharris@whoopdedo.org>
391
 * @author <vpribish at shopping dot com>
392
 * @link   http://www.php.net/manual/en/function.utf8-decode.php
142
 * faster replacement for utf8_entities_to_umlauts()
143
 * not all features of utf8_entities_to_umlauts() --> utf8_unhtml() are supported!
144
 * @author thorn
393 145
 */
394
function utf8_tohtml ($str) {
395
    $ret = '';
396
    foreach (utf8_to_unicode($str) as $cp) {
397
        if ($cp < 0x80)
398
            $ret .= chr($cp);
399
        //elseif ($cp < 0x100)
400
        //    $ret .= "&#$cp;";
401
        else
402
            $ret .= "&#$cp;";
403
        //    $ret .= '&#x'.dechex($cp).';';
404
    }
405
    return $ret;
146
function utf8_fast_entities_to_umlauts($str) {
147
	if(UTF8_MBSTRING) {
148
		// we need this for use with mb_convert_encoding
149
		$str = str_replace(array('&amp;','&gt;','&lt;','&quot;','&#39;','&nbsp;'), array('&amp;amp;','&amp;gt;','&amp;lt;','&amp;quot;','&amp;#39;','&amp;nbsp;'), $str);
150
		// we need two mb_convert_encoding()-calls - is this a bug?
151
		// mb_convert_encoding("ö&ouml;", 'UTF-8', 'HTML-ENTITIES'); // with string in utf-8-encoding doesn't work. Result: "öö"
152
		// Work-around: convert all umlauts to entities first ("ö&ouml;"->"&ouml;&ouml;"), then all entities to umlauts ("&ouml;&ouml;"->"öö")
153
		return(mb_convert_encoding(mb_convert_encoding($str, 'HTML-ENTITIES', 'UTF-8'),'UTF-8', 'HTML-ENTITIES'));
154
	} else {
155
		global $named_entities;global $numbered_entities;
156
		$str = str_replace($named_entities, $numbered_entities, $str);
157
		$str = preg_replace("/&#([0-9]+);/e", "code_to_utf8($1)", $str);
158
	}
159
	return($str);
406 160
}
407

  
408
/*
409
 * Decodes HTML entities to UTF-8 characters
410
 *
411
 * Convert any &#..; entity to a codepoint,
412
 * The entities flag defaults to only decoding numeric entities.
413
 * Pass HTML_ENTITIES and named entities, including &amp; &lt; etc.
414
 * are handled as well. Avoids the problem that would occur if you
415
 * had to decode "&amp;#38;&#38;amp;#38;"
416
 *
417
 * unhtmlspecialchars(utf8_unhtml($s)) -> "&#38;&#38;"
418
 * utf8_unhtml(unhtmlspecialchars($s)) -> "&&amp#38;"
419
 * what it should be                   -> "&#38;&amp#38;"
420
 *
421
 * @author Tom N Harris <tnharris@whoopdedo.org>
422
 * @param  string  $str      UTF-8 encoded string
423
 * @param  boolean $entities Flag controlling decoding of named entities.
424
 * @return UTF-8 encoded string with numeric (and named) entities replaced.
425
 */
426
function utf8_unhtml($str, $entities=null) {
427
    static $decoder = null;
428
    if (is_null($decoder))
429
      $decoder = new utf8_entity_decoder();
430
    if (is_null($entities))
431
        return preg_replace_callback('/(&#([Xx])?([0-9A-Za-z]+);)/m',
432
                                     'utf8_decode_numeric', $str);
433
    else
434
        return preg_replace_callback('/&(#)?([Xx])?([0-9A-Za-z]+);/m',
435
                                     array(&$decoder, 'decode'), $str);
161
// support-function for utf8_fast_entities_to_umlauts()
162
function code_to_utf8($num) {
163
	if ($num <= 0x7F) {
164
		return chr($num);
165
	} elseif ($num <= 0x7FF) {
166
		return chr(($num >> 6) + 192) . chr(($num & 63) + 128);
167
	} elseif ($num <= 0xFFFF) {
168
		 return chr(($num >> 12) + 224) . chr((($num >> 6) & 63) + 128) . chr(($num & 63) + 128);
169
	} elseif ($num <= 0x1FFFFF) {
170
		return chr(($num >> 18) + 240) . chr((($num >> 12) & 63) + 128) . chr((($num >> 6) & 63) + 128) . chr(($num & 63) + 128);
171
	}
172
	return "?";
436 173
}
437
function utf8_decode_numeric($ent) {
438
    switch ($ent[2]) {
439
      case 'X':
440
      case 'x':
441
          $cp = hexdec($ent[3]);
442
          break;
443
      default:
444
          $cp = intval($ent[3]);
445
          break;
446
    }
447
    return unicode_to_utf8(array($cp));
448
}
449
class utf8_entity_decoder {
450
    var $table;
451
    function utf8_entity_decoder() {
452
        $table = get_html_translation_table(HTML_ENTITIES);
453
        $table = array_flip($table);
454
        $this->table = array_map(array(&$this,'makeutf8'), $table);
455
    }
456
    function makeutf8($c) {
457
        return unicode_to_utf8(array(ord($c)));
458
    }
459
    function decode($ent) {
460
        if ($ent[1] == '#') {
461
            return utf8_decode_numeric($ent);
462
        } elseif (array_key_exists($ent[0],$this->table)) {
463
            return $this->table[$ent[0]];
464
        } else {
465
            return $ent[0];
466
        }
467
    }
468
}
469 174

  
470 175
/*
471
 * Takes an UTF-8 string and returns an array of ints representing the
472
 * Unicode characters. Astral planes are supported ie. the ints in the
473
 * output can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates
474
 * are not allowed.
475
 *
476
 * If $strict is set to true the function returns false if the input
477
 * string isn't a valid UTF-8 octet sequence and raises a PHP error at
478
 * level E_USER_WARNING
479
 *
480
 * Note: this function has been modified slightly in this library to
481
 * trigger errors on encountering bad bytes
482
 *
483
 * @author <hsivonen@iki.fi>
484
 * @author Harry Fuecks <hfuecks@gmail.com>
485
 * @param  string  UTF-8 encoded string
486
 * @param  boolean Check for invalid sequences?
487
 * @return mixed array of unicode code points or false if UTF-8 invalid
488
 * @see    unicode_to_utf8
489
 * @link   http://hsivonen.iki.fi/php-utf8/
490
 * @link   http://sourceforge.net/projects/phputf8/
176
 * faster replacement for utf8_umlauts_to_entities()
177
 * not all features of utf8_umlauts_to_entities() --> utf8_tohtml() are supported!
178
 * @author thorn
491 179
 */
492
function utf8_to_unicode($str,$strict=false) {
493
    $mState = 0;     // cached expected number of octets after the current octet
494
                     // until the beginning of the next UTF8 character sequence
495
    $mUcs4  = 0;     // cached Unicode character
496
    $mBytes = 1;     // cached expected number of octets in the current sequence
497

  
498
    $out = array();
499

  
500
    $len = strlen($str);
501

  
502
    for($i = 0; $i < $len; $i++) {
503

  
504
        $in = ord($str{$i});
505

  
506
        if ( $mState == 0) {
507

  
508
            // When mState is zero we expect either a US-ASCII character or a
509
            // multi-octet sequence.
510
            if (0 == (0x80 & ($in))) {
511
                // US-ASCII, pass straight through.
512
                $out[] = $in;
513
                $mBytes = 1;
514

  
515
            } else if (0xC0 == (0xE0 & ($in))) {
516
                // First octet of 2 octet sequence
517
                $mUcs4 = ($in);
518
                $mUcs4 = ($mUcs4 & 0x1F) << 6;
519
                $mState = 1;
520
                $mBytes = 2;
521

  
522
            } else if (0xE0 == (0xF0 & ($in))) {
523
                // First octet of 3 octet sequence
524
                $mUcs4 = ($in);
525
                $mUcs4 = ($mUcs4 & 0x0F) << 12;
526
                $mState = 2;
527
                $mBytes = 3;
528

  
529
            } else if (0xF0 == (0xF8 & ($in))) {
530
                // First octet of 4 octet sequence
531
                $mUcs4 = ($in);
532
                $mUcs4 = ($mUcs4 & 0x07) << 18;
533
                $mState = 3;
534
                $mBytes = 4;
535

  
536
            } else if (0xF8 == (0xFC & ($in))) {
537
                /* First octet of 5 octet sequence.
538
                 *
539
                 * This is illegal because the encoded codepoint must be either
540
                 * (a) not the shortest form or
541
                 * (b) outside the Unicode range of 0-0x10FFFF.
542
                 * Rather than trying to resynchronize, we will carry on until the end
543
                 * of the sequence and let the later error handling code catch it.
544
                 */
545
                $mUcs4 = ($in);
546
                $mUcs4 = ($mUcs4 & 0x03) << 24;
547
                $mState = 4;
548
                $mBytes = 5;
549

  
550
            } else if (0xFC == (0xFE & ($in))) {
551
                // First octet of 6 octet sequence, see comments for 5 octet sequence.
552
                $mUcs4 = ($in);
553
                $mUcs4 = ($mUcs4 & 1) << 30;
554
                $mState = 5;
555
                $mBytes = 6;
556

  
557
            } elseif($strict) {
558
                /* Current octet is neither in the US-ASCII range nor a legal first
559
                 * octet of a multi-octet sequence.
560
                 */
561
                trigger_error(
562
                        'utf8_to_unicode: Illegal sequence identifier '.
563
                            'in UTF-8 at byte '.$i,
564
                        E_USER_WARNING
565
                    );
566
                return false;
567

  
568
            }
569

  
570
        } else {
571

  
572
            // When mState is non-zero, we expect a continuation of the multi-octet
573
            // sequence
574
            if (0x80 == (0xC0 & ($in))) {
575

  
576
                // Legal continuation.
577
                $shift = ($mState - 1) * 6;
578
                $tmp = $in;
579
                $tmp = ($tmp & 0x0000003F) << $shift;
580
                $mUcs4 |= $tmp;
581

  
582
                /*
583
                 * End of the multi-octet sequence. mUcs4 now contains the final
584
                 * Unicode codepoint to be output
585
                 */
586
                if (0 == --$mState) {
587

  
588
                    /*
589
                     * Check for illegal sequences and codepoints.
590
                     */
591
                    // From Unicode 3.1, non-shortest form is illegal
592
                    if (((2 == $mBytes) && ($mUcs4 < 0x0080)) ||
593
                        ((3 == $mBytes) && ($mUcs4 < 0x0800)) ||
594
                        ((4 == $mBytes) && ($mUcs4 < 0x10000)) ||
595
                        (4 < $mBytes) ||
596
                        // From Unicode 3.2, surrogate characters are illegal
597
                        (($mUcs4 & 0xFFFFF800) == 0xD800) ||
598
                        // Codepoints outside the Unicode range are illegal
599
                        ($mUcs4 > 0x10FFFF)) {
600

  
601
                        if($strict){
602
                            trigger_error(
603
                                    'utf8_to_unicode: Illegal sequence or codepoint '.
604
                                        'in UTF-8 at byte '.$i,
605
                                    E_USER_WARNING
606
                                );
607

  
608
                            return false;
609
                        }
610

  
611
                    }
612

  
613
                    if (0xFEFF != $mUcs4) {
614
                        // BOM is legal but we don't want to output it
615
                        $out[] = $mUcs4;
616
                    }
617

  
618
                    //initialize UTF8 cache
619
                    $mState = 0;
620
                    $mUcs4  = 0;
621
                    $mBytes = 1;
622
                }
623

  
624
            } elseif($strict) {
625
                /*
626
                 *((0xC0 & (*in) != 0x80) && (mState != 0))
627
                 * Incomplete multi-octet sequence.
628
                 */
629
                trigger_error(
630
                        'utf8_to_unicode: Incomplete multi-octet '.
631
                        '   sequence in UTF-8 at byte '.$i,
632
                        E_USER_WARNING
633
                    );
634

  
635
                return false;
636
            }
637
        }
638
    }
639
    return $out;
180
function utf8_fast_umlauts_to_entities($string, $named_entities=true) {
181
	if(UTF8_MBSTRING)
182
		return(mb_convert_encoding($string, 'HTML-ENTITIES', 'UTF-8'));
183
	else {
184
		global $named_entities;global $numbered_entities;
185
		$new = "";
186
		$i=0;
187
		$len=strlen($string);
188
		if($len==0) return $string;
189
		do {
190
			if(ord($string{$i}) <= 127) $ud = $string{$i++};
191
			elseif(ord($string{$i}) <= 223) $ud = (ord($string{$i++})-192)*64 + (ord($string{$i++})-128);
192
			elseif(ord($string{$i}) <= 239) $ud = (ord($string{$i++})-224)*4096 + (ord($string{$i++})-128)*64 + (ord($string{$i++})-128);
193
			elseif(ord($string{$i}) <= 247) $ud = (ord($string{$i++})-240)*262144 + (ord($string{$i++})-128)*4096 + (ord($string{$i++})-128)*64 + (ord($string{$i++})-128);
194
			else $ud = ord($string{$i++}); // error!
195
			if($ud > 127) {
196
				$new .= "&#$ud;";
197
			} else {
198
				$new .= $ud;
199
			}
200
		} while($i < $len);
201
		$string = $new;
202
		if($named_entities)
203
			$string = str_replace($numbered_entities, $named_entities, $string);
204
	}
205
	return($string);
640 206
}
641 207

  
642 208
/*
643
 * Takes an array of ints representing the Unicode characters and returns
644
 * a UTF-8 string. Astral planes are supported ie. the ints in the
645
 * input can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates
646
 * are not allowed.
647
 *
648
 * If $strict is set to true the function returns false if the input
649
 * array contains ints that represent surrogates or are outside the
650
 * Unicode range and raises a PHP error at level E_USER_WARNING
651
 *
652
 * Note: this function has been modified slightly in this library to use
653
 * output buffering to concatenate the UTF-8 string (faster) as well as
654
 * reference the array by it's keys
655
 *
656
 * @param  array of unicode code points representing a string
657
 * @param  boolean Check for invalid sequences?
658
 * @return mixed UTF-8 string or false if array contains invalid code points
659
 * @author <hsivonen@iki.fi>
660
 * @author Harry Fuecks <hfuecks@gmail.com>
661
 * @see    utf8_to_unicode
662
 * @link   http://hsivonen.iki.fi/php-utf8/
663
 * @link   http://sourceforge.net/projects/phputf8/
664
 */
665
function unicode_to_utf8($arr,$strict=false) {
666
    if (!is_array($arr)) return '';
667
    ob_start();
668

  
669
    foreach (array_keys($arr) as $k) {
670

  
671
        # ASCII range (including control chars)
672
        if ( ($arr[$k] >= 0) && ($arr[$k] <= 0x007f) ) {
673

  
674
            echo chr($arr[$k]);
675

  
676
        # 2 byte sequence
677
        } else if ($arr[$k] <= 0x07ff) {
678

  
679
            echo chr(0xc0 | ($arr[$k] >> 6));
680
            echo chr(0x80 | ($arr[$k] & 0x003f));
681

  
682
        # Byte order mark (skip)
683
        } else if($arr[$k] == 0xFEFF) {
684

  
685
            // nop -- zap the BOM
686

  
687
        # Test for illegal surrogates
688
        } else if ($arr[$k] >= 0xD800 && $arr[$k] <= 0xDFFF) {
689

  
690
            // found a surrogate
691
            if($strict){
692
                trigger_error(
693
                    'unicode_to_utf8: Illegal surrogate '.
694
                        'at index: '.$k.', value: '.$arr[$k],
695
                    E_USER_WARNING
696
                    );
697
                return false;
698
            }
699

  
700
        # 3 byte sequence
701
        } else if ($arr[$k] <= 0xffff) {
702

  
703
            echo chr(0xe0 | ($arr[$k] >> 12));
704
            echo chr(0x80 | (($arr[$k] >> 6) & 0x003f));
705
            echo chr(0x80 | ($arr[$k] & 0x003f));
706

  
707
        # 4 byte sequence
708
        } else if ($arr[$k] <= 0x10ffff) {
709

  
710
            echo chr(0xf0 | ($arr[$k] >> 18));
711
            echo chr(0x80 | (($arr[$k] >> 12) & 0x3f));
712
            echo chr(0x80 | (($arr[$k] >> 6) & 0x3f));
713
            echo chr(0x80 | ($arr[$k] & 0x3f));
714

  
715
        } elseif($strict) {
716

  
717
            trigger_error(
718
                'unicode_to_utf8: Codepoint out of Unicode range '.
719
                    'at index: '.$k.', value: '.$arr[$k],
720
                E_USER_WARNING
721
                );
722

  
723
            // out of range
724
            return false;
725
        }
726
    }
727

  
728
    $result = ob_get_contents();
729
    ob_end_clean();
730
    return $result;
731
}
732

  
733
/*
734
 * Replace bad bytes with an alternative character
735
 *
736
 * ASCII character is recommended for replacement char
737
 *
738
 * PCRE Pattern to locate bad bytes in a UTF-8 string
739
 * Comes from W3 FAQ: Multilingual Forms
740
 * Note: modified to include full ASCII range including control chars
741
 *
742
 * @author Harry Fuecks <hfuecks@gmail.com>
743
 * @see http://www.w3.org/International/questions/qa-forms-utf-8
744
 * @param string to search
745
 * @param string to replace bad bytes with (defaults to '?') - use ASCII
746
 * @return string
747
 */
748
function utf8_bad_replace($str, $replace = '') {
749
    $UTF8_BAD =
750
     '([\x00-\x7F]'.                          # ASCII (including control chars)
751
     '|[\xC2-\xDF][\x80-\xBF]'.               # non-overlong 2-byte
752
     '|\xE0[\xA0-\xBF][\x80-\xBF]'.           # excluding overlongs
753
     '|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'.    # straight 3-byte
754
     '|\xED[\x80-\x9F][\x80-\xBF]'.           # excluding surrogates
755
     '|\xF0[\x90-\xBF][\x80-\xBF]{2}'.        # planes 1-3
756
     '|[\xF1-\xF3][\x80-\xBF]{3}'.            # planes 4-15
757
     '|\xF4[\x80-\x8F][\x80-\xBF]{2}'.        # plane 16
758
     '|(.{1}))';                              # invalid byte
759
    ob_start();
760
    while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches)) {
761
        if ( !isset($matches[2])) {
762
            echo $matches[0];
763
        } else {
764
            echo $replace;
765
        }
766
        $str = substr($str,strlen($matches[0]));
767
    }
768
    $result = ob_get_contents();
769
    ob_end_clean();
770
    return $result;
771
}
772

  
773
/*
774
 * URL-Encode a filename to allow unicodecharacters
775
 *
776
 * Slashes are not encoded
777
 *
778
 * When the second parameter is true the string will
779
 * be encoded only if non ASCII characters are detected -
780
 * This makes it safe to run it multiple times on the
781
 * same string (default is true)
782
 *
783
 * @author Andreas Gohr <andi@splitbrain.org>
784
 * @see    urlencode
785
 */
786
function utf8_encodeFN($file,$safe=true){
787
  if($safe && preg_match('#^[a-zA-Z0-9/_\-.%]+$#',$file)){
788
    return $file;
789
  }
790
  $file = urlencode($file);
791
  $file = str_replace('%2F','/',$file);
792
  return $file;
793
}
794

  
795
/*
796
 * URL-Decode a filename
797
 *
798
 * This is just a wrapper around urldecode
799
 *
800
 * @author Andreas Gohr <andi@splitbrain.org>
801
 * @see    urldecode
802
 */
803
function utf8_decodeFN($file){
804
  $file = urldecode($file);
805
  return $file;
806
}
807

  
808
/*
809
 * Moved some functions from framework/functions.php to here - thorn
810
 */
811

  
812
/*
813
 * Decode HTML entities to UTF-8 characters
814
 * 
815
 * Will replace all numeric and named entities, except
816
 * &gt; &lt; &apos; &quot; &#39; &nbsp;
817
 * 
818
 * @param  string UTF-8 or ASCII encoded string
819
 * @return string UTF-8 encoded string with numeric and named entities replaced.
820
 */
821
function utf8_entities_to_umlauts($str) {
822
	global $named_to_numbered_entities;
823
	// we have to prevent "&#39;" from beeing decoded
824
	$str = str_replace("&#39;", "&_#39;", $str);
825
	$str = strtr($str, $named_to_numbered_entities);
826
	$str = utf8_unhtml($str);
827
	$str = str_replace("&_#39;", "&#39;", $str);
828

  
829
	return($str);
830
}
831

  
832
/*
833
 * Encode UTF-8 characters to HTML entities
834
 *
835
 * Will replace all UTF-8 encoded characters to numeric/named entities
836
 *
837
 * @param  string UTF-8 encoded string
838
 * @param  bool Replace numbered by named entities
839
 * @return string ASCII encoded string with all UTF-8 characters replaced by numeric/named entities
840
 */
841
function utf8_umlauts_to_entities($str, $named_entities=true) {
842
	global $numbered_to_named_entities;
843
	$str = utf8_tohtml($str);
844
	if($named_entities)
845
		$str = strtr($str, $numbered_to_named_entities);
846
	return($str);
847
}
848

  
849
/*
850 209
 * Converts from various charsets to UTF-8
851 210
 *
852 211
 * Will convert a string from various charsets to UTF-8.
853
 * HTML-entities will be converted, too.
212
 * HTML-entities may be converted, too.
854 213
 * In case of error the returned string is unchanged, and a message is emitted.
855 214
 * Supported charsets are:
856 215
 * direct: iso_8859_1 iso_8859_2 iso_8859_3 iso_8859_4 iso_8859_5
......
862 221
 * @param  string  The charset to convert from, defaults to DEFAULT_CHARSET
863 222
 * @return string  A string in UTF-8-encoding, with all entities decoded, too.
864 223
 *                 String is unchanged in case of error.
224
 * @author thorn
865 225
 */
866
function charset_to_utf8($str, $charset_in=DEFAULT_CHARSET) {
226
function charset_to_utf8($str, $charset_in=DEFAULT_CHARSET, $decode_entities=true) {
867 227
	global $iso_8859_2_to_utf8, $iso_8859_3_to_utf8, $iso_8859_4_to_utf8, $iso_8859_5_to_utf8, $iso_8859_6_to_utf8, $iso_8859_7_to_utf8, $iso_8859_8_to_utf8, $iso_8859_9_to_utf8, $iso_8859_10_to_utf8, $iso_8859_11_to_utf8;
868 228
	$charset_in = strtoupper($charset_in);
869 229
	if ($charset_in == "") { $charset_in = 'UTF-8'; }
......
882 242
	// check if we have UTF-8 or a plain ASCII string
883 243
	if($charset_in == 'UTF-8' || utf8_isASCII($str)) {
884 244
		// we have utf-8. Just replace HTML-entities and return
885
		if(preg_match('/&[#0-9a-zA-Z]+;/',$str))
886
			return(utf8_entities_to_umlauts($str));
245
		if($decode_entities && preg_match('/&[#0-9a-zA-Z]+;/',$str))
246
			return(utf8_fast_entities_to_umlauts($str));
887 247
		else // nothing to do
888 248
			return($str);
889 249
	}
......
920 280
	}
921 281
	if($converted) {
922 282
		// we have utf-8, now replace HTML-entities and return
923
		if(preg_match('/&[#0-9a-zA-Z]+;/',$str))
924
			$str = utf8_entities_to_umlauts($str);
925
		// just to be sure, replace bad characters
926
		$str = utf8_bad_replace($str, '?');
283
		if($decode_entities && preg_match('/&[#0-9a-zA-Z]+;/',$str))
284
			$str = utf8_fast_entities_to_umlauts($str);
927 285
		return($str);
928 286
	}
929 287
	
......
940 298
 * Converts from UTF-8 to various charsets
941 299
 *
942 300
 * Will convert a string from UTF-8 to various charsets.
943
 * HTML-entities will be converted, too.
301
 * HTML-entities will not! be converted.
944 302
 * In case of error the returned string is unchanged, and a message is emitted.
945 303
 * Supported charsets are:
946 304
 * direct: iso_8859_1 iso_8859_2 iso_8859_3 iso_8859_4 iso_8859_5
......
952 310
 * @param  string  The charset to convert to, defaults to DEFAULT_CHARSET
953 311
 * @return string  A string in a supported encoding, with all entities decoded, too.
954 312
 *                 String is unchanged in case of error.
313
 * @author thorn
955 314
 */
956 315
function utf8_to_charset($str, $charset_out=DEFAULT_CHARSET) {
957 316
	global $utf8_to_iso_8859_2, $utf8_to_iso_8859_3, $utf8_to_iso_8859_4, $utf8_to_iso_8859_5, $utf8_to_iso_8859_6, $utf8_to_iso_8859_7, $utf8_to_iso_8859_8, $utf8_to_iso_8859_9, $utf8_to_iso_8859_10, $utf8_to_iso_8859_11;
......
968 327
		return($str);
969 328
	}
970 329
	
330
	// the string comes from charset_to_utf8(), so we can skip this
971 331
	// replace HTML-entities first
972
	if(preg_match('/&[#0-9a-zA-Z]+;/',$str))
973
		$str = utf8_entities_to_umlauts($str);
332
	//if(preg_match('/&[#0-9a-zA-Z]+;/',$str))
333
	//	$str = utf8_entities_to_umlauts($str);
974 334
	
975 335
	// check if we need to convert
976 336
	if($charset_out == 'UTF-8' || utf8_isASCII($str)) {
......
1030 390
 *
1031 391
 * @param  string  Filename to convert (all encodings from charset_to_utf8() are allowed)
1032 392
 * @return string  ASCII encoded string, to use as filename in wb's page_filename() and media_filename
393
 * @author thorn
1033 394
 */
1034 395
function entities_to_7bit($str) {
1035 396
	// convert to UTF-8
......
1042 403
	$str = utf8_romanize($str);
1043 404
	// missed some? - Many UTF-8-chars can't be romanized
1044 405
	// convert to HTML-entities, and replace entites by hex-numbers
1045
	$str = utf8_umlauts_to_entities($str, false);
406
	$str = utf8_fast_umlauts_to_entities($str, false);
1046 407
	$str = str_replace('&#39;', '&apos;', $str);
1047 408
	$str = preg_replace('/&#([0-9]+);/e', "dechex('$1')",  $str);
1048 409
	// maybe there are some &gt; &lt; &apos; &quot; &amp; &nbsp; left, replace them too
......
1057 418
 * 
1058 419
 * Will replace all numeric and named entities except
1059 420
 * &gt; &lt; &apos; &quot; &#39; &nbsp;
1060
 * In case of error the returned string is unchanged, and a message is emitted.
1061
 * Supported charsets are:
1062
 * direct: iso_8859_1 iso_8859_2 iso_8859_3 iso_8859_4 iso_8859_5
1063
 *         iso_8859_6 iso_8859_7 iso_8859_8 iso_8859_9 iso_8859_10 iso_8859_11
1064
 * mb_convert_encoding: all wb charsets (except those from 'direct'); but not GB2312
1065
 * iconv:  all wb charsets (except those from 'direct')
1066
 * 
1067
 * @param  string  A string in DEFAULT_CHARSET encoding
1068
 * @return string  A string in $charset_out encoding with numeric and named entities replaced.
1069
 *         The string is unchanged in case of error. 
421
 * @author thorn
1070 422
 */
1071 423
function entities_to_umlauts2($string, $charset_out=DEFAULT_CHARSET) {
1072
	$string = charset_to_utf8($string, DEFAULT_CHARSET);
1073
	//if(utf8_check($string)) // this is to much time-consuming
424
	$string = charset_to_utf8($string, DEFAULT_CHARSET, true);
425
	//if(utf8_check($string)) // this check is to much time-consuming (this may fail only if AddDefaultCharset is set)
1074 426
		$string = utf8_to_charset($string, $charset_out);
1075 427
	return ($string);
1076 428
}
......
1079 431
 * Convert a string from mixed html-entities/umlauts to pure ASCII with HTML-entities
1080 432
 * 
1081 433
 * Will convert a string in $charset_in encoding to a pure ASCII string with HTML-entities.
1082
 * In case of error the returned string is unchanged, and a message is emitted.
1083
 * Supported charsets are:
1084
 * direct: iso_8859_1 iso_8859_2 iso_8859_3 iso_8859_4 iso_8859_5
1085
 *         iso_8859_6 iso_8859_7 iso_8859_8 iso_8859_9 iso_8859_10 iso_8859_11
1086
 * mb_convert_encoding: all wb charsets (except those from 'direct'); but not GB2312
1087
 * iconv:  all wb charsets (except those from 'direct')
1088
 * 
1089
 * @param  string  A string in $charset_in encoding
1090
 * @return string  A string in ASCII encoding with numeric and named entities.
1091
 *         The string is unchanged in case of error. 
434
 * @author thorn
1092 435
 */
1093 436
function umlauts_to_entities2($string, $charset_in=DEFAULT_CHARSET) {
1094
	$string = charset_to_utf8($string, $charset_in);
1095
	//if(utf8_check($string)) // this is to much time-consuming
1096
		$string = utf8_umlauts_to_entities($string);
437
	$string = charset_to_utf8($string, $charset_in, false);
438
	//if(utf8_check($string)) // this check is to much time-consuming (this may fail only if AddDefaultCharset is set)
439
		$string = utf8_fast_umlauts_to_entities($string, false);
1097 440
	return($string);
1098 441
}
1099 442

  

Also available in: Unified diff