Revision 761
Added by thorn over 17 years ago
| functions-utf8.php | ||
|---|---|---|
| 24 | 24 |
*/ |
| 25 | 25 |
|
| 26 | 26 |
/* |
| 27 |
* A large part of this file is based on 'utf8.php' from the DokuWiki-project.
|
|
| 27 |
* A part of this file is based on 'utf8.php' from the DokuWiki-project. |
|
| 28 | 28 |
* (http://www.splitbrain.org/projects/dokuwiki): |
| 29 | 29 |
** |
| 30 | 30 |
* UTF8 helper functions |
| ... | ... | |
| 33 | 33 |
** |
| 34 | 34 |
* modified for use with Website Baker |
| 35 | 35 |
* from thorn, Jan. 2008 |
| 36 |
* |
|
| 37 |
* most of the original functions appeared to be to slow with large strings, so i replaced them with my own ones |
|
| 38 |
* thorn, Mar. 2008 |
|
| 36 | 39 |
*/ |
| 37 | 40 |
|
| 38 | 41 |
// Functions we use in Website Baker: |
| ... | ... | |
| 48 | 51 |
/* |
| 49 | 52 |
* check for mb_string support |
| 50 | 53 |
*/ |
| 54 |
//define('UTF8_NOMBSTRING',1); // uncomment this to forbid use of mb_string-functions
|
|
| 51 | 55 |
if(!defined('UTF8_MBSTRING')){
|
| 52 | 56 |
if(function_exists('mb_substr') && !defined('UTF8_NOMBSTRING')){
|
| 53 | 57 |
define('UTF8_MBSTRING',1);
|
| ... | ... | |
| 73 | 77 |
} |
| 74 | 78 |
|
| 75 | 79 |
/* |
| 76 |
* Strips all highbyte chars |
|
| 77 |
* |
|
| 78 |
* Returns a pure ASCII7 string |
|
| 79 |
* |
|
| 80 |
* @author Andreas Gohr <andi@splitbrain.org> |
|
| 81 |
*/ |
|
| 82 |
function utf8_strip($str){
|
|
| 83 |
$ascii = ''; |
|
| 84 |
for($i=0; $i<strlen($str); $i++){
|
|
| 85 |
if(ord($str{$i}) <128){
|
|
| 86 |
$ascii .= $str{$i};
|
|
| 87 |
} |
|
| 88 |
} |
|
| 89 |
return $ascii; |
|
| 90 |
} |
|
| 91 |
|
|
| 92 |
/* |
|
| 93 | 80 |
* Tries to detect if a string is in Unicode encoding |
| 94 | 81 |
* |
| 95 | 82 |
* @author <bmorel@ssi.fr> |
| ... | ... | |
| 114 | 101 |
} |
| 115 | 102 |
|
| 116 | 103 |
/* |
| 117 |
* Unicode aware replacement for strlen() |
|
| 118 |
* |
|
| 119 |
* utf8_decode() converts characters that are not in ISO-8859-1 |
|
| 120 |
* to '?', which, for the purpose of counting, is alright - It's |
|
| 121 |
* even faster than mb_strlen. |
|
| 122 |
* |
|
| 123 |
* @author <chernyshevsky at hotmail dot com> |
|
| 124 |
* @see strlen() |
|
| 125 |
* @see utf8_decode() |
|
| 126 |
*/ |
|
| 127 |
function utf8_strlen($string){
|
|
| 128 |
return strlen(utf8_decode($string)); |
|
| 129 |
} |
|
| 130 |
|
|
| 131 |
/* |
|
| 132 |
* UTF-8 aware alternative to substr |
|
| 133 |
* |
|
| 134 |
* Return part of a string given character offset (and optionally length) |
|
| 135 |
* |
|
| 136 |
* @author Harry Fuecks <hfuecks@gmail.com> |
|
| 137 |
* @author Chris Smith <chris@jalakai.co.uk> |
|
| 138 |
* @param string |
|
| 139 |
* @param integer number of UTF-8 characters offset (from left) |
|
| 140 |
* @param integer (optional) length in UTF-8 characters from offset |
|
| 141 |
* @return mixed string or false if failure |
|
| 142 |
*/ |
|
| 143 |
function utf8_substr($str, $offset, $length = null) {
|
|
| 144 |
if(UTF8_MBSTRING){
|
|
| 145 |
if( $length === null ){
|
|
| 146 |
return mb_substr($str, $offset); |
|
| 147 |
}else{
|
|
| 148 |
return mb_substr($str, $offset, $length); |
|
| 149 |
} |
|
| 150 |
} |
|
| 151 |
|
|
| 152 |
/* |
|
| 153 |
* Notes: |
|
| 154 |
* |
|
| 155 |
* no mb string support, so we'll use pcre regex's with 'u' flag |
|
| 156 |
* pcre only supports repetitions of less than 65536, in order to accept up to MAXINT values for |
|
| 157 |
* offset and length, we'll repeat a group of 65535 characters when needed (ok, up to MAXINT-65536) |
|
| 158 |
* |
|
| 159 |
* substr documentation states false can be returned in some cases (e.g. offset > string length) |
|
| 160 |
* mb_substr never returns false, it will return an empty string instead. |
|
| 161 |
* |
|
| 162 |
* calculating the number of characters in the string is a relatively expensive operation, so |
|
| 163 |
* we only carry it out when necessary. It isn't necessary for +ve offsets and no specified length |
|
| 164 |
*/ |
|
| 165 |
|
|
| 166 |
// cast parameters to appropriate types to avoid multiple notices/warnings |
|
| 167 |
$str = (string)$str; // generates E_NOTICE for PHP4 objects, but not PHP5 objects |
|
| 168 |
$offset = (int)$offset; |
|
| 169 |
if (!is_null($length)) $length = (int)$length; |
|
| 170 |
|
|
| 171 |
// handle trivial cases |
|
| 172 |
if ($length === 0) return ''; |
|
| 173 |
if ($offset < 0 && $length < 0 && $length < $offset) return ''; |
|
| 174 |
|
|
| 175 |
$offset_pattern = ''; |
|
| 176 |
$length_pattern = ''; |
|
| 177 |
|
|
| 178 |
// normalise -ve offsets (we could use a tail anchored pattern, but they are horribly slow!) |
|
| 179 |
if ($offset < 0) {
|
|
| 180 |
$strlen = strlen(utf8_decode($str)); // see notes |
|
| 181 |
$offset = $strlen + $offset; |
|
| 182 |
if ($offset < 0) $offset = 0; |
|
| 183 |
} |
|
| 184 |
|
|
| 185 |
// establish a pattern for offset, a non-captured group equal in length to offset |
|
| 186 |
if ($offset > 0) {
|
|
| 187 |
$Ox = (int)($offset/65535); |
|
| 188 |
$Oy = $offset%65535; |
|
| 189 |
|
|
| 190 |
if ($Ox) $offset_pattern = '(?:.{65535}){'.$Ox.'}';
|
|
| 191 |
$offset_pattern = '^(?:'.$offset_pattern.'.{'.$Oy.'})';
|
|
| 192 |
} else {
|
|
| 193 |
$offset_pattern = '^'; // offset == 0; just anchor the pattern |
|
| 194 |
} |
|
| 195 |
|
|
| 196 |
// establish a pattern for length |
|
| 197 |
if (is_null($length)) {
|
|
| 198 |
$length_pattern = '(.*)$'; // the rest of the string |
|
| 199 |
} else {
|
|
| 200 |
|
|
| 201 |
if (!isset($strlen)) $strlen = strlen(utf8_decode($str)); // see notes |
|
| 202 |
if ($offset > $strlen) return ''; // another trivial case |
|
| 203 |
|
|
| 204 |
if ($length > 0) {
|
|
| 205 |
|
|
| 206 |
$length = min($strlen-$offset, $length); // reduce any length that would go passed the end of the string |
|
| 207 |
|
|
| 208 |
$Lx = (int)($length/65535); |
|
| 209 |
$Ly = $length%65535; |
|
| 210 |
|
|
| 211 |
// +ve length requires ... a captured group of length characters |
|
| 212 |
if ($Lx) $length_pattern = '(?:.{65535}){'.$Lx.'}';
|
|
| 213 |
$length_pattern = '('.$length_pattern.'.{'.$Ly.'})';
|
|
| 214 |
|
|
| 215 |
} else if ($length < 0) {
|
|
| 216 |
|
|
| 217 |
if ($length < ($offset - $strlen)) return ''; |
|
| 218 |
|
|
| 219 |
$Lx = (int)((-$length)/65535); |
|
| 220 |
$Ly = (-$length)%65535; |
|
| 221 |
|
|
| 222 |
// -ve length requires ... capture everything except a group of -length characters |
|
| 223 |
// anchored at the tail-end of the string |
|
| 224 |
if ($Lx) $length_pattern = '(?:.{65535}){'.$Lx.'}';
|
|
| 225 |
$length_pattern = '(.*)(?:'.$length_pattern.'.{'.$Ly.'})$';
|
|
| 226 |
} |
|
| 227 |
} |
|
| 228 |
|
|
| 229 |
if (!preg_match('#'.$offset_pattern.$length_pattern.'#us',$str,$match)) return '';
|
|
| 230 |
return $match[1]; |
|
| 231 |
} |
|
| 232 |
|
|
| 233 |
/* |
|
| 234 |
* Unicode aware replacement for substr_replace() |
|
| 235 |
* |
|
| 236 |
* @author Andreas Gohr <andi@splitbrain.org> |
|
| 237 |
* @see substr_replace() |
|
| 238 |
*/ |
|
| 239 |
function utf8_substr_replace($string, $replacement, $start , $length=0 ){
|
|
| 240 |
$ret = ''; |
|
| 241 |
if($start>0) $ret .= utf8_substr($string, 0, $start); |
|
| 242 |
$ret .= $replacement; |
|
| 243 |
$ret .= utf8_substr($string, $start+$length); |
|
| 244 |
return $ret; |
|
| 245 |
} |
|
| 246 |
|
|
| 247 |
/* |
|
| 248 |
* Unicode aware replacement for ltrim() |
|
| 249 |
* |
|
| 250 |
* @author Andreas Gohr <andi@splitbrain.org> |
|
| 251 |
* @see ltrim() |
|
| 252 |
* @return string |
|
| 253 |
*/ |
|
| 254 |
function utf8_ltrim($str,$charlist=''){
|
|
| 255 |
if($charlist == '') return ltrim($str); |
|
| 256 |
|
|
| 257 |
//quote charlist for use in a characterclass |
|
| 258 |
$charlist = preg_replace('!([\\\\\\-\\]\\[/])!','\\\${1}',$charlist);
|
|
| 259 |
|
|
| 260 |
return preg_replace('/^['.$charlist.']+/u','',$str);
|
|
| 261 |
} |
|
| 262 |
|
|
| 263 |
/* |
|
| 264 |
* Unicode aware replacement for rtrim() |
|
| 265 |
* |
|
| 266 |
* @author Andreas Gohr <andi@splitbrain.org> |
|
| 267 |
* @see rtrim() |
|
| 268 |
* @return string |
|
| 269 |
*/ |
|
| 270 |
function utf8_rtrim($str,$charlist=''){
|
|
| 271 |
if($charlist == '') return rtrim($str); |
|
| 272 |
|
|
| 273 |
//quote charlist for use in a characterclass |
|
| 274 |
$charlist = preg_replace('!([\\\\\\-\\]\\[/])!','\\\${1}',$charlist);
|
|
| 275 |
|
|
| 276 |
return preg_replace('/['.$charlist.']+$/u','',$str);
|
|
| 277 |
} |
|
| 278 |
|
|
| 279 |
/* |
|
| 280 |
* Unicode aware replacement for trim() |
|
| 281 |
* |
|
| 282 |
* @author Andreas Gohr <andi@splitbrain.org> |
|
| 283 |
* @see trim() |
|
| 284 |
* @return string |
|
| 285 |
*/ |
|
| 286 |
function utf8_trim($str,$charlist='') {
|
|
| 287 |
if($charlist == '') return trim($str); |
|
| 288 |
|
|
| 289 |
return utf8_ltrim(utf8_rtrim($str,$charlist),$charlist); |
|
| 290 |
} |
|
| 291 |
|
|
| 292 |
/* |
|
| 293 |
* This is a unicode aware replacement for strtolower() |
|
| 294 |
* |
|
| 295 |
* Uses mb_string extension if available |
|
| 296 |
* |
|
| 297 |
* @author Leo Feyer <leo@typolight.org> |
|
| 298 |
* @see strtolower() |
|
| 299 |
* @see utf8_strtoupper() |
|
| 300 |
*/ |
|
| 301 |
function utf8_strtolower($string){
|
|
| 302 |
if(UTF8_MBSTRING) return mb_strtolower($string,'utf-8'); |
|
| 303 |
|
|
| 304 |
global $UTF8_UPPER_TO_LOWER; |
|
| 305 |
return strtr($string,$UTF8_UPPER_TO_LOWER); |
|
| 306 |
} |
|
| 307 |
|
|
| 308 |
/* |
|
| 309 |
* This is a unicode aware replacement for strtoupper() |
|
| 310 |
* |
|
| 311 |
* Uses mb_string extension if available |
|
| 312 |
* |
|
| 313 |
* @author Leo Feyer <leo@typolight.org> |
|
| 314 |
* @see strtoupper() |
|
| 315 |
* @see utf8_strtoupper() |
|
| 316 |
*/ |
|
| 317 |
function utf8_strtoupper($string){
|
|
| 318 |
if(UTF8_MBSTRING) return mb_strtoupper($string,'utf-8'); |
|
| 319 |
|
|
| 320 |
global $UTF8_LOWER_TO_UPPER; |
|
| 321 |
return strtr($string,$UTF8_LOWER_TO_UPPER); |
|
| 322 |
} |
|
| 323 |
|
|
| 324 |
/* |
|
| 325 | 104 |
* Romanize a non-latin string |
| 326 | 105 |
* |
| 327 | 106 |
* @author Andreas Gohr <andi@splitbrain.org> |
| ... | ... | |
| 356 | 135 |
} |
| 357 | 136 |
|
| 358 | 137 |
/* |
| 359 |
* This is an Unicode aware replacement for strpos |
|
| 360 |
* |
|
| 361 |
* @author Leo Feyer <leo@typolight.org> |
|
| 362 |
* @see strpos() |
|
| 363 |
* @param string |
|
| 364 |
* @param string |
|
| 365 |
* @param integer |
|
| 366 |
* @return integer |
|
| 138 |
* added functions - thorn |
|
| 367 | 139 |
*/ |
| 368 |
function utf8_strpos($haystack, $needle, $offset=0){
|
|
| 369 |
$comp = 0; |
|
| 370 |
$length = null; |
|
| 371 | 140 |
|
| 372 |
while (is_null($length) || $length < $offset) {
|
|
| 373 |
$pos = strpos($haystack, $needle, $offset + $comp); |
|
| 374 |
|
|
| 375 |
if ($pos === false) |
|
| 376 |
return false; |
|
| 377 |
|
|
| 378 |
$length = utf8_strlen(substr($haystack, 0, $pos)); |
|
| 379 |
|
|
| 380 |
if ($length < $offset) |
|
| 381 |
$comp = $pos - $length; |
|
| 382 |
} |
|
| 383 |
|
|
| 384 |
return $length; |
|
| 385 |
} |
|
| 386 |
|
|
| 387 | 141 |
/* |
| 388 |
* Encodes UTF-8 characters to HTML entities |
|
| 389 |
* |
|
| 390 |
* @author Tom N Harris <tnharris@whoopdedo.org> |
|
| 391 |
* @author <vpribish at shopping dot com> |
|
| 392 |
* @link http://www.php.net/manual/en/function.utf8-decode.php |
|
| 142 |
* faster replacement for utf8_entities_to_umlauts() |
|
| 143 |
* not all features of utf8_entities_to_umlauts() --> utf8_unhtml() are supported! |
|
| 144 |
* @author thorn |
|
| 393 | 145 |
*/ |
| 394 |
function utf8_tohtml ($str) {
|
|
| 395 |
$ret = ''; |
|
| 396 |
foreach (utf8_to_unicode($str) as $cp) {
|
|
| 397 |
if ($cp < 0x80) |
|
| 398 |
$ret .= chr($cp); |
|
| 399 |
//elseif ($cp < 0x100) |
|
| 400 |
// $ret .= "&#$cp;"; |
|
| 401 |
else |
|
| 402 |
$ret .= "&#$cp;"; |
|
| 403 |
// $ret .= '&#x'.dechex($cp).';'; |
|
| 404 |
} |
|
| 405 |
return $ret; |
|
| 146 |
function utf8_fast_entities_to_umlauts($str) {
|
|
| 147 |
if(UTF8_MBSTRING) {
|
|
| 148 |
// we need this for use with mb_convert_encoding |
|
| 149 |
$str = str_replace(array('&','>','<','"',''',' '), array('&amp;','&gt;','&lt;','&quot;','&#39;','&nbsp;'), $str);
|
|
| 150 |
// we need two mb_convert_encoding()-calls - is this a bug? |
|
| 151 |
// mb_convert_encoding("öö", 'UTF-8', 'HTML-ENTITIES'); // with string in utf-8-encoding doesn't work. Result: "öö"
|
|
| 152 |
// Work-around: convert all umlauts to entities first ("öö"->"öö"), then all entities to umlauts ("öö"->"öö")
|
|
| 153 |
return(mb_convert_encoding(mb_convert_encoding($str, 'HTML-ENTITIES', 'UTF-8'),'UTF-8', 'HTML-ENTITIES')); |
|
| 154 |
} else {
|
|
| 155 |
global $named_entities;global $numbered_entities; |
|
| 156 |
$str = str_replace($named_entities, $numbered_entities, $str); |
|
| 157 |
$str = preg_replace("/&#([0-9]+);/e", "code_to_utf8($1)", $str);
|
|
| 158 |
} |
|
| 159 |
return($str); |
|
| 406 | 160 |
} |
| 407 |
|
|
| 408 |
/* |
|
| 409 |
* Decodes HTML entities to UTF-8 characters |
|
| 410 |
* |
|
| 411 |
* Convert any &#..; entity to a codepoint, |
|
| 412 |
* The entities flag defaults to only decoding numeric entities. |
|
| 413 |
* Pass HTML_ENTITIES and named entities, including & < etc. |
|
| 414 |
* are handled as well. Avoids the problem that would occur if you |
|
| 415 |
* had to decode "&#38;&amp;#38;" |
|
| 416 |
* |
|
| 417 |
* unhtmlspecialchars(utf8_unhtml($s)) -> "&&" |
|
| 418 |
* utf8_unhtml(unhtmlspecialchars($s)) -> "&&#38;" |
|
| 419 |
* what it should be -> "&&#38;" |
|
| 420 |
* |
|
| 421 |
* @author Tom N Harris <tnharris@whoopdedo.org> |
|
| 422 |
* @param string $str UTF-8 encoded string |
|
| 423 |
* @param boolean $entities Flag controlling decoding of named entities. |
|
| 424 |
* @return UTF-8 encoded string with numeric (and named) entities replaced. |
|
| 425 |
*/ |
|
| 426 |
function utf8_unhtml($str, $entities=null) {
|
|
| 427 |
static $decoder = null; |
|
| 428 |
if (is_null($decoder)) |
|
| 429 |
$decoder = new utf8_entity_decoder(); |
|
| 430 |
if (is_null($entities)) |
|
| 431 |
return preg_replace_callback('/(&#([Xx])?([0-9A-Za-z]+);)/m',
|
|
| 432 |
'utf8_decode_numeric', $str); |
|
| 433 |
else |
|
| 434 |
return preg_replace_callback('/&(#)?([Xx])?([0-9A-Za-z]+);/m',
|
|
| 435 |
array(&$decoder, 'decode'), $str); |
|
| 161 |
// support-function for utf8_fast_entities_to_umlauts() |
|
| 162 |
function code_to_utf8($num) {
|
|
| 163 |
if ($num <= 0x7F) {
|
|
| 164 |
return chr($num); |
|
| 165 |
} elseif ($num <= 0x7FF) {
|
|
| 166 |
return chr(($num >> 6) + 192) . chr(($num & 63) + 128); |
|
| 167 |
} elseif ($num <= 0xFFFF) {
|
|
| 168 |
return chr(($num >> 12) + 224) . chr((($num >> 6) & 63) + 128) . chr(($num & 63) + 128); |
|
| 169 |
} elseif ($num <= 0x1FFFFF) {
|
|
| 170 |
return chr(($num >> 18) + 240) . chr((($num >> 12) & 63) + 128) . chr((($num >> 6) & 63) + 128) . chr(($num & 63) + 128); |
|
| 171 |
} |
|
| 172 |
return "?"; |
|
| 436 | 173 |
} |
| 437 |
function utf8_decode_numeric($ent) {
|
|
| 438 |
switch ($ent[2]) {
|
|
| 439 |
case 'X': |
|
| 440 |
case 'x': |
|
| 441 |
$cp = hexdec($ent[3]); |
|
| 442 |
break; |
|
| 443 |
default: |
|
| 444 |
$cp = intval($ent[3]); |
|
| 445 |
break; |
|
| 446 |
} |
|
| 447 |
return unicode_to_utf8(array($cp)); |
|
| 448 |
} |
|
| 449 |
class utf8_entity_decoder {
|
|
| 450 |
var $table; |
|
| 451 |
function utf8_entity_decoder() {
|
|
| 452 |
$table = get_html_translation_table(HTML_ENTITIES); |
|
| 453 |
$table = array_flip($table); |
|
| 454 |
$this->table = array_map(array(&$this,'makeutf8'), $table); |
|
| 455 |
} |
|
| 456 |
function makeutf8($c) {
|
|
| 457 |
return unicode_to_utf8(array(ord($c))); |
|
| 458 |
} |
|
| 459 |
function decode($ent) {
|
|
| 460 |
if ($ent[1] == '#') {
|
|
| 461 |
return utf8_decode_numeric($ent); |
|
| 462 |
} elseif (array_key_exists($ent[0],$this->table)) {
|
|
| 463 |
return $this->table[$ent[0]]; |
|
| 464 |
} else {
|
|
| 465 |
return $ent[0]; |
|
| 466 |
} |
|
| 467 |
} |
|
| 468 |
} |
|
| 469 | 174 |
|
| 470 | 175 |
/* |
| 471 |
* Takes an UTF-8 string and returns an array of ints representing the |
|
| 472 |
* Unicode characters. Astral planes are supported ie. the ints in the |
|
| 473 |
* output can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates |
|
| 474 |
* are not allowed. |
|
| 475 |
* |
|
| 476 |
* If $strict is set to true the function returns false if the input |
|
| 477 |
* string isn't a valid UTF-8 octet sequence and raises a PHP error at |
|
| 478 |
* level E_USER_WARNING |
|
| 479 |
* |
|
| 480 |
* Note: this function has been modified slightly in this library to |
|
| 481 |
* trigger errors on encountering bad bytes |
|
| 482 |
* |
|
| 483 |
* @author <hsivonen@iki.fi> |
|
| 484 |
* @author Harry Fuecks <hfuecks@gmail.com> |
|
| 485 |
* @param string UTF-8 encoded string |
|
| 486 |
* @param boolean Check for invalid sequences? |
|
| 487 |
* @return mixed array of unicode code points or false if UTF-8 invalid |
|
| 488 |
* @see unicode_to_utf8 |
|
| 489 |
* @link http://hsivonen.iki.fi/php-utf8/ |
|
| 490 |
* @link http://sourceforge.net/projects/phputf8/ |
|
| 176 |
* faster replacement for utf8_umlauts_to_entities() |
|
| 177 |
* not all features of utf8_umlauts_to_entities() --> utf8_tohtml() are supported! |
|
| 178 |
* @author thorn |
|
| 491 | 179 |
*/ |
| 492 |
function utf8_to_unicode($str,$strict=false) {
|
|
| 493 |
$mState = 0; // cached expected number of octets after the current octet |
|
| 494 |
// until the beginning of the next UTF8 character sequence |
|
| 495 |
$mUcs4 = 0; // cached Unicode character |
|
| 496 |
$mBytes = 1; // cached expected number of octets in the current sequence |
|
| 497 |
|
|
| 498 |
$out = array(); |
|
| 499 |
|
|
| 500 |
$len = strlen($str); |
|
| 501 |
|
|
| 502 |
for($i = 0; $i < $len; $i++) {
|
|
| 503 |
|
|
| 504 |
$in = ord($str{$i});
|
|
| 505 |
|
|
| 506 |
if ( $mState == 0) {
|
|
| 507 |
|
|
| 508 |
// When mState is zero we expect either a US-ASCII character or a |
|
| 509 |
// multi-octet sequence. |
|
| 510 |
if (0 == (0x80 & ($in))) {
|
|
| 511 |
// US-ASCII, pass straight through. |
|
| 512 |
$out[] = $in; |
|
| 513 |
$mBytes = 1; |
|
| 514 |
|
|
| 515 |
} else if (0xC0 == (0xE0 & ($in))) {
|
|
| 516 |
// First octet of 2 octet sequence |
|
| 517 |
$mUcs4 = ($in); |
|
| 518 |
$mUcs4 = ($mUcs4 & 0x1F) << 6; |
|
| 519 |
$mState = 1; |
|
| 520 |
$mBytes = 2; |
|
| 521 |
|
|
| 522 |
} else if (0xE0 == (0xF0 & ($in))) {
|
|
| 523 |
// First octet of 3 octet sequence |
|
| 524 |
$mUcs4 = ($in); |
|
| 525 |
$mUcs4 = ($mUcs4 & 0x0F) << 12; |
|
| 526 |
$mState = 2; |
|
| 527 |
$mBytes = 3; |
|
| 528 |
|
|
| 529 |
} else if (0xF0 == (0xF8 & ($in))) {
|
|
| 530 |
// First octet of 4 octet sequence |
|
| 531 |
$mUcs4 = ($in); |
|
| 532 |
$mUcs4 = ($mUcs4 & 0x07) << 18; |
|
| 533 |
$mState = 3; |
|
| 534 |
$mBytes = 4; |
|
| 535 |
|
|
| 536 |
} else if (0xF8 == (0xFC & ($in))) {
|
|
| 537 |
/* First octet of 5 octet sequence. |
|
| 538 |
* |
|
| 539 |
* This is illegal because the encoded codepoint must be either |
|
| 540 |
* (a) not the shortest form or |
|
| 541 |
* (b) outside the Unicode range of 0-0x10FFFF. |
|
| 542 |
* Rather than trying to resynchronize, we will carry on until the end |
|
| 543 |
* of the sequence and let the later error handling code catch it. |
|
| 544 |
*/ |
|
| 545 |
$mUcs4 = ($in); |
|
| 546 |
$mUcs4 = ($mUcs4 & 0x03) << 24; |
|
| 547 |
$mState = 4; |
|
| 548 |
$mBytes = 5; |
|
| 549 |
|
|
| 550 |
} else if (0xFC == (0xFE & ($in))) {
|
|
| 551 |
// First octet of 6 octet sequence, see comments for 5 octet sequence. |
|
| 552 |
$mUcs4 = ($in); |
|
| 553 |
$mUcs4 = ($mUcs4 & 1) << 30; |
|
| 554 |
$mState = 5; |
|
| 555 |
$mBytes = 6; |
|
| 556 |
|
|
| 557 |
} elseif($strict) {
|
|
| 558 |
/* Current octet is neither in the US-ASCII range nor a legal first |
|
| 559 |
* octet of a multi-octet sequence. |
|
| 560 |
*/ |
|
| 561 |
trigger_error( |
|
| 562 |
'utf8_to_unicode: Illegal sequence identifier '. |
|
| 563 |
'in UTF-8 at byte '.$i, |
|
| 564 |
E_USER_WARNING |
|
| 565 |
); |
|
| 566 |
return false; |
|
| 567 |
|
|
| 568 |
} |
|
| 569 |
|
|
| 570 |
} else {
|
|
| 571 |
|
|
| 572 |
// When mState is non-zero, we expect a continuation of the multi-octet |
|
| 573 |
// sequence |
|
| 574 |
if (0x80 == (0xC0 & ($in))) {
|
|
| 575 |
|
|
| 576 |
// Legal continuation. |
|
| 577 |
$shift = ($mState - 1) * 6; |
|
| 578 |
$tmp = $in; |
|
| 579 |
$tmp = ($tmp & 0x0000003F) << $shift; |
|
| 580 |
$mUcs4 |= $tmp; |
|
| 581 |
|
|
| 582 |
/* |
|
| 583 |
* End of the multi-octet sequence. mUcs4 now contains the final |
|
| 584 |
* Unicode codepoint to be output |
|
| 585 |
*/ |
|
| 586 |
if (0 == --$mState) {
|
|
| 587 |
|
|
| 588 |
/* |
|
| 589 |
* Check for illegal sequences and codepoints. |
|
| 590 |
*/ |
|
| 591 |
// From Unicode 3.1, non-shortest form is illegal |
|
| 592 |
if (((2 == $mBytes) && ($mUcs4 < 0x0080)) || |
|
| 593 |
((3 == $mBytes) && ($mUcs4 < 0x0800)) || |
|
| 594 |
((4 == $mBytes) && ($mUcs4 < 0x10000)) || |
|
| 595 |
(4 < $mBytes) || |
|
| 596 |
// From Unicode 3.2, surrogate characters are illegal |
|
| 597 |
(($mUcs4 & 0xFFFFF800) == 0xD800) || |
|
| 598 |
// Codepoints outside the Unicode range are illegal |
|
| 599 |
($mUcs4 > 0x10FFFF)) {
|
|
| 600 |
|
|
| 601 |
if($strict){
|
|
| 602 |
trigger_error( |
|
| 603 |
'utf8_to_unicode: Illegal sequence or codepoint '. |
|
| 604 |
'in UTF-8 at byte '.$i, |
|
| 605 |
E_USER_WARNING |
|
| 606 |
); |
|
| 607 |
|
|
| 608 |
return false; |
|
| 609 |
} |
|
| 610 |
|
|
| 611 |
} |
|
| 612 |
|
|
| 613 |
if (0xFEFF != $mUcs4) {
|
|
| 614 |
// BOM is legal but we don't want to output it |
|
| 615 |
$out[] = $mUcs4; |
|
| 616 |
} |
|
| 617 |
|
|
| 618 |
//initialize UTF8 cache |
|
| 619 |
$mState = 0; |
|
| 620 |
$mUcs4 = 0; |
|
| 621 |
$mBytes = 1; |
|
| 622 |
} |
|
| 623 |
|
|
| 624 |
} elseif($strict) {
|
|
| 625 |
/* |
|
| 626 |
*((0xC0 & (*in) != 0x80) && (mState != 0)) |
|
| 627 |
* Incomplete multi-octet sequence. |
|
| 628 |
*/ |
|
| 629 |
trigger_error( |
|
| 630 |
'utf8_to_unicode: Incomplete multi-octet '. |
|
| 631 |
' sequence in UTF-8 at byte '.$i, |
|
| 632 |
E_USER_WARNING |
|
| 633 |
); |
|
| 634 |
|
|
| 635 |
return false; |
|
| 636 |
} |
|
| 637 |
} |
|
| 638 |
} |
|
| 639 |
return $out; |
|
| 180 |
function utf8_fast_umlauts_to_entities($string, $named_entities=true) {
|
|
| 181 |
if(UTF8_MBSTRING) |
|
| 182 |
return(mb_convert_encoding($string, 'HTML-ENTITIES', 'UTF-8')); |
|
| 183 |
else {
|
|
| 184 |
global $named_entities;global $numbered_entities; |
|
| 185 |
$new = ""; |
|
| 186 |
$i=0; |
|
| 187 |
$len=strlen($string); |
|
| 188 |
if($len==0) return $string; |
|
| 189 |
do {
|
|
| 190 |
if(ord($string{$i}) <= 127) $ud = $string{$i++};
|
|
| 191 |
elseif(ord($string{$i}) <= 223) $ud = (ord($string{$i++})-192)*64 + (ord($string{$i++})-128);
|
|
| 192 |
elseif(ord($string{$i}) <= 239) $ud = (ord($string{$i++})-224)*4096 + (ord($string{$i++})-128)*64 + (ord($string{$i++})-128);
|
|
| 193 |
elseif(ord($string{$i}) <= 247) $ud = (ord($string{$i++})-240)*262144 + (ord($string{$i++})-128)*4096 + (ord($string{$i++})-128)*64 + (ord($string{$i++})-128);
|
|
| 194 |
else $ud = ord($string{$i++}); // error!
|
|
| 195 |
if($ud > 127) {
|
|
| 196 |
$new .= "&#$ud;"; |
|
| 197 |
} else {
|
|
| 198 |
$new .= $ud; |
|
| 199 |
} |
|
| 200 |
} while($i < $len); |
|
| 201 |
$string = $new; |
|
| 202 |
if($named_entities) |
|
| 203 |
$string = str_replace($numbered_entities, $named_entities, $string); |
|
| 204 |
} |
|
| 205 |
return($string); |
|
| 640 | 206 |
} |
| 641 | 207 |
|
| 642 | 208 |
/* |
| 643 |
* Takes an array of ints representing the Unicode characters and returns |
|
| 644 |
* a UTF-8 string. Astral planes are supported ie. the ints in the |
|
| 645 |
* input can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates |
|
| 646 |
* are not allowed. |
|
| 647 |
* |
|
| 648 |
* If $strict is set to true the function returns false if the input |
|
| 649 |
* array contains ints that represent surrogates or are outside the |
|
| 650 |
* Unicode range and raises a PHP error at level E_USER_WARNING |
|
| 651 |
* |
|
| 652 |
* Note: this function has been modified slightly in this library to use |
|
| 653 |
* output buffering to concatenate the UTF-8 string (faster) as well as |
|
| 654 |
* reference the array by it's keys |
|
| 655 |
* |
|
| 656 |
* @param array of unicode code points representing a string |
|
| 657 |
* @param boolean Check for invalid sequences? |
|
| 658 |
* @return mixed UTF-8 string or false if array contains invalid code points |
|
| 659 |
* @author <hsivonen@iki.fi> |
|
| 660 |
* @author Harry Fuecks <hfuecks@gmail.com> |
|
| 661 |
* @see utf8_to_unicode |
|
| 662 |
* @link http://hsivonen.iki.fi/php-utf8/ |
|
| 663 |
* @link http://sourceforge.net/projects/phputf8/ |
|
| 664 |
*/ |
|
| 665 |
function unicode_to_utf8($arr,$strict=false) {
|
|
| 666 |
if (!is_array($arr)) return ''; |
|
| 667 |
ob_start(); |
|
| 668 |
|
|
| 669 |
foreach (array_keys($arr) as $k) {
|
|
| 670 |
|
|
| 671 |
# ASCII range (including control chars) |
|
| 672 |
if ( ($arr[$k] >= 0) && ($arr[$k] <= 0x007f) ) {
|
|
| 673 |
|
|
| 674 |
echo chr($arr[$k]); |
|
| 675 |
|
|
| 676 |
# 2 byte sequence |
|
| 677 |
} else if ($arr[$k] <= 0x07ff) {
|
|
| 678 |
|
|
| 679 |
echo chr(0xc0 | ($arr[$k] >> 6)); |
|
| 680 |
echo chr(0x80 | ($arr[$k] & 0x003f)); |
|
| 681 |
|
|
| 682 |
# Byte order mark (skip) |
|
| 683 |
} else if($arr[$k] == 0xFEFF) {
|
|
| 684 |
|
|
| 685 |
// nop -- zap the BOM |
|
| 686 |
|
|
| 687 |
# Test for illegal surrogates |
|
| 688 |
} else if ($arr[$k] >= 0xD800 && $arr[$k] <= 0xDFFF) {
|
|
| 689 |
|
|
| 690 |
// found a surrogate |
|
| 691 |
if($strict){
|
|
| 692 |
trigger_error( |
|
| 693 |
'unicode_to_utf8: Illegal surrogate '. |
|
| 694 |
'at index: '.$k.', value: '.$arr[$k], |
|
| 695 |
E_USER_WARNING |
|
| 696 |
); |
|
| 697 |
return false; |
|
| 698 |
} |
|
| 699 |
|
|
| 700 |
# 3 byte sequence |
|
| 701 |
} else if ($arr[$k] <= 0xffff) {
|
|
| 702 |
|
|
| 703 |
echo chr(0xe0 | ($arr[$k] >> 12)); |
|
| 704 |
echo chr(0x80 | (($arr[$k] >> 6) & 0x003f)); |
|
| 705 |
echo chr(0x80 | ($arr[$k] & 0x003f)); |
|
| 706 |
|
|
| 707 |
# 4 byte sequence |
|
| 708 |
} else if ($arr[$k] <= 0x10ffff) {
|
|
| 709 |
|
|
| 710 |
echo chr(0xf0 | ($arr[$k] >> 18)); |
|
| 711 |
echo chr(0x80 | (($arr[$k] >> 12) & 0x3f)); |
|
| 712 |
echo chr(0x80 | (($arr[$k] >> 6) & 0x3f)); |
|
| 713 |
echo chr(0x80 | ($arr[$k] & 0x3f)); |
|
| 714 |
|
|
| 715 |
} elseif($strict) {
|
|
| 716 |
|
|
| 717 |
trigger_error( |
|
| 718 |
'unicode_to_utf8: Codepoint out of Unicode range '. |
|
| 719 |
'at index: '.$k.', value: '.$arr[$k], |
|
| 720 |
E_USER_WARNING |
|
| 721 |
); |
|
| 722 |
|
|
| 723 |
// out of range |
|
| 724 |
return false; |
|
| 725 |
} |
|
| 726 |
} |
|
| 727 |
|
|
| 728 |
$result = ob_get_contents(); |
|
| 729 |
ob_end_clean(); |
|
| 730 |
return $result; |
|
| 731 |
} |
|
| 732 |
|
|
| 733 |
/* |
|
| 734 |
* Replace bad bytes with an alternative character |
|
| 735 |
* |
|
| 736 |
* ASCII character is recommended for replacement char |
|
| 737 |
* |
|
| 738 |
* PCRE Pattern to locate bad bytes in a UTF-8 string |
|
| 739 |
* Comes from W3 FAQ: Multilingual Forms |
|
| 740 |
* Note: modified to include full ASCII range including control chars |
|
| 741 |
* |
|
| 742 |
* @author Harry Fuecks <hfuecks@gmail.com> |
|
| 743 |
* @see http://www.w3.org/International/questions/qa-forms-utf-8 |
|
| 744 |
* @param string to search |
|
| 745 |
* @param string to replace bad bytes with (defaults to '?') - use ASCII |
|
| 746 |
* @return string |
|
| 747 |
*/ |
|
| 748 |
function utf8_bad_replace($str, $replace = '') {
|
|
| 749 |
$UTF8_BAD = |
|
| 750 |
'([\x00-\x7F]'. # ASCII (including control chars) |
|
| 751 |
'|[\xC2-\xDF][\x80-\xBF]'. # non-overlong 2-byte |
|
| 752 |
'|\xE0[\xA0-\xBF][\x80-\xBF]'. # excluding overlongs |
|
| 753 |
'|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # straight 3-byte
|
|
| 754 |
'|\xED[\x80-\x9F][\x80-\xBF]'. # excluding surrogates |
|
| 755 |
'|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # planes 1-3
|
|
| 756 |
'|[\xF1-\xF3][\x80-\xBF]{3}'. # planes 4-15
|
|
| 757 |
'|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # plane 16
|
|
| 758 |
'|(.{1}))'; # invalid byte
|
|
| 759 |
ob_start(); |
|
| 760 |
while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches)) {
|
|
| 761 |
if ( !isset($matches[2])) {
|
|
| 762 |
echo $matches[0]; |
|
| 763 |
} else {
|
|
| 764 |
echo $replace; |
|
| 765 |
} |
|
| 766 |
$str = substr($str,strlen($matches[0])); |
|
| 767 |
} |
|
| 768 |
$result = ob_get_contents(); |
|
| 769 |
ob_end_clean(); |
|
| 770 |
return $result; |
|
| 771 |
} |
|
| 772 |
|
|
| 773 |
/* |
|
| 774 |
* URL-Encode a filename to allow unicodecharacters |
|
| 775 |
* |
|
| 776 |
* Slashes are not encoded |
|
| 777 |
* |
|
| 778 |
* When the second parameter is true the string will |
|
| 779 |
* be encoded only if non ASCII characters are detected - |
|
| 780 |
* This makes it safe to run it multiple times on the |
|
| 781 |
* same string (default is true) |
|
| 782 |
* |
|
| 783 |
* @author Andreas Gohr <andi@splitbrain.org> |
|
| 784 |
* @see urlencode |
|
| 785 |
*/ |
|
| 786 |
function utf8_encodeFN($file,$safe=true){
|
|
| 787 |
if($safe && preg_match('#^[a-zA-Z0-9/_\-.%]+$#',$file)){
|
|
| 788 |
return $file; |
|
| 789 |
} |
|
| 790 |
$file = urlencode($file); |
|
| 791 |
$file = str_replace('%2F','/',$file);
|
|
| 792 |
return $file; |
|
| 793 |
} |
|
| 794 |
|
|
| 795 |
/* |
|
| 796 |
* URL-Decode a filename |
|
| 797 |
* |
|
| 798 |
* This is just a wrapper around urldecode |
|
| 799 |
* |
|
| 800 |
* @author Andreas Gohr <andi@splitbrain.org> |
|
| 801 |
* @see urldecode |
|
| 802 |
*/ |
|
| 803 |
function utf8_decodeFN($file){
|
|
| 804 |
$file = urldecode($file); |
|
| 805 |
return $file; |
|
| 806 |
} |
|
| 807 |
|
|
| 808 |
/* |
|
| 809 |
* Moved some functions from framework/functions.php to here - thorn |
|
| 810 |
*/ |
|
| 811 |
|
|
| 812 |
/* |
|
| 813 |
* Decode HTML entities to UTF-8 characters |
|
| 814 |
* |
|
| 815 |
* Will replace all numeric and named entities, except |
|
| 816 |
* > < ' " ' |
|
| 817 |
* |
|
| 818 |
* @param string UTF-8 or ASCII encoded string |
|
| 819 |
* @return string UTF-8 encoded string with numeric and named entities replaced. |
|
| 820 |
*/ |
|
| 821 |
function utf8_entities_to_umlauts($str) {
|
|
| 822 |
global $named_to_numbered_entities; |
|
| 823 |
// we have to prevent "'" from beeing decoded |
|
| 824 |
$str = str_replace("'", "&_#39;", $str);
|
|
| 825 |
$str = strtr($str, $named_to_numbered_entities); |
|
| 826 |
$str = utf8_unhtml($str); |
|
| 827 |
$str = str_replace("&_#39;", "'", $str);
|
|
| 828 |
|
|
| 829 |
return($str); |
|
| 830 |
} |
|
| 831 |
|
|
| 832 |
/* |
|
| 833 |
* Encode UTF-8 characters to HTML entities |
|
| 834 |
* |
|
| 835 |
* Will replace all UTF-8 encoded characters to numeric/named entities |
|
| 836 |
* |
|
| 837 |
* @param string UTF-8 encoded string |
|
| 838 |
* @param bool Replace numbered by named entities |
|
| 839 |
* @return string ASCII encoded string with all UTF-8 characters replaced by numeric/named entities |
|
| 840 |
*/ |
|
| 841 |
function utf8_umlauts_to_entities($str, $named_entities=true) {
|
|
| 842 |
global $numbered_to_named_entities; |
|
| 843 |
$str = utf8_tohtml($str); |
|
| 844 |
if($named_entities) |
|
| 845 |
$str = strtr($str, $numbered_to_named_entities); |
|
| 846 |
return($str); |
|
| 847 |
} |
|
| 848 |
|
|
| 849 |
/* |
|
| 850 | 209 |
* Converts from various charsets to UTF-8 |
| 851 | 210 |
* |
| 852 | 211 |
* Will convert a string from various charsets to UTF-8. |
| 853 |
* HTML-entities will be converted, too.
|
|
| 212 |
* HTML-entities may be converted, too.
|
|
| 854 | 213 |
* In case of error the returned string is unchanged, and a message is emitted. |
| 855 | 214 |
* Supported charsets are: |
| 856 | 215 |
* direct: iso_8859_1 iso_8859_2 iso_8859_3 iso_8859_4 iso_8859_5 |
| ... | ... | |
| 862 | 221 |
* @param string The charset to convert from, defaults to DEFAULT_CHARSET |
| 863 | 222 |
* @return string A string in UTF-8-encoding, with all entities decoded, too. |
| 864 | 223 |
* String is unchanged in case of error. |
| 224 |
* @author thorn |
|
| 865 | 225 |
*/ |
| 866 |
function charset_to_utf8($str, $charset_in=DEFAULT_CHARSET) {
|
|
| 226 |
function charset_to_utf8($str, $charset_in=DEFAULT_CHARSET, $decode_entities=true) {
|
|
| 867 | 227 |
global $iso_8859_2_to_utf8, $iso_8859_3_to_utf8, $iso_8859_4_to_utf8, $iso_8859_5_to_utf8, $iso_8859_6_to_utf8, $iso_8859_7_to_utf8, $iso_8859_8_to_utf8, $iso_8859_9_to_utf8, $iso_8859_10_to_utf8, $iso_8859_11_to_utf8; |
| 868 | 228 |
$charset_in = strtoupper($charset_in); |
| 869 | 229 |
if ($charset_in == "") { $charset_in = 'UTF-8'; }
|
| ... | ... | |
| 882 | 242 |
// check if we have UTF-8 or a plain ASCII string |
| 883 | 243 |
if($charset_in == 'UTF-8' || utf8_isASCII($str)) {
|
| 884 | 244 |
// we have utf-8. Just replace HTML-entities and return |
| 885 |
if(preg_match('/&[#0-9a-zA-Z]+;/',$str))
|
|
| 886 |
return(utf8_entities_to_umlauts($str)); |
|
| 245 |
if($decode_entities && preg_match('/&[#0-9a-zA-Z]+;/',$str))
|
|
| 246 |
return(utf8_fast_entities_to_umlauts($str));
|
|
| 887 | 247 |
else // nothing to do |
| 888 | 248 |
return($str); |
| 889 | 249 |
} |
| ... | ... | |
| 920 | 280 |
} |
| 921 | 281 |
if($converted) {
|
| 922 | 282 |
// we have utf-8, now replace HTML-entities and return |
| 923 |
if(preg_match('/&[#0-9a-zA-Z]+;/',$str))
|
|
| 924 |
$str = utf8_entities_to_umlauts($str); |
|
| 925 |
// just to be sure, replace bad characters |
|
| 926 |
$str = utf8_bad_replace($str, '?'); |
|
| 283 |
if($decode_entities && preg_match('/&[#0-9a-zA-Z]+;/',$str))
|
|
| 284 |
$str = utf8_fast_entities_to_umlauts($str); |
|
| 927 | 285 |
return($str); |
| 928 | 286 |
} |
| 929 | 287 |
|
| ... | ... | |
| 940 | 298 |
* Converts from UTF-8 to various charsets |
| 941 | 299 |
* |
| 942 | 300 |
* Will convert a string from UTF-8 to various charsets. |
| 943 |
* HTML-entities will be converted, too.
|
|
| 301 |
* HTML-entities will not! be converted.
|
|
| 944 | 302 |
* In case of error the returned string is unchanged, and a message is emitted. |
| 945 | 303 |
* Supported charsets are: |
| 946 | 304 |
* direct: iso_8859_1 iso_8859_2 iso_8859_3 iso_8859_4 iso_8859_5 |
| ... | ... | |
| 952 | 310 |
* @param string The charset to convert to, defaults to DEFAULT_CHARSET |
| 953 | 311 |
* @return string A string in a supported encoding, with all entities decoded, too. |
| 954 | 312 |
* String is unchanged in case of error. |
| 313 |
* @author thorn |
|
| 955 | 314 |
*/ |
| 956 | 315 |
function utf8_to_charset($str, $charset_out=DEFAULT_CHARSET) {
|
| 957 | 316 |
global $utf8_to_iso_8859_2, $utf8_to_iso_8859_3, $utf8_to_iso_8859_4, $utf8_to_iso_8859_5, $utf8_to_iso_8859_6, $utf8_to_iso_8859_7, $utf8_to_iso_8859_8, $utf8_to_iso_8859_9, $utf8_to_iso_8859_10, $utf8_to_iso_8859_11; |
| ... | ... | |
| 968 | 327 |
return($str); |
| 969 | 328 |
} |
| 970 | 329 |
|
| 330 |
// the string comes from charset_to_utf8(), so we can skip this |
|
| 971 | 331 |
// replace HTML-entities first |
| 972 |
if(preg_match('/&[#0-9a-zA-Z]+;/',$str))
|
|
| 973 |
$str = utf8_entities_to_umlauts($str); |
|
| 332 |
//if(preg_match('/&[#0-9a-zA-Z]+;/',$str))
|
|
| 333 |
// $str = utf8_entities_to_umlauts($str);
|
|
| 974 | 334 |
|
| 975 | 335 |
// check if we need to convert |
| 976 | 336 |
if($charset_out == 'UTF-8' || utf8_isASCII($str)) {
|
| ... | ... | |
| 1030 | 390 |
* |
| 1031 | 391 |
* @param string Filename to convert (all encodings from charset_to_utf8() are allowed) |
| 1032 | 392 |
* @return string ASCII encoded string, to use as filename in wb's page_filename() and media_filename |
| 393 |
* @author thorn |
|
| 1033 | 394 |
*/ |
| 1034 | 395 |
function entities_to_7bit($str) {
|
| 1035 | 396 |
// convert to UTF-8 |
| ... | ... | |
| 1042 | 403 |
$str = utf8_romanize($str); |
| 1043 | 404 |
// missed some? - Many UTF-8-chars can't be romanized |
| 1044 | 405 |
// convert to HTML-entities, and replace entites by hex-numbers |
| 1045 |
$str = utf8_umlauts_to_entities($str, false); |
|
| 406 |
$str = utf8_fast_umlauts_to_entities($str, false);
|
|
| 1046 | 407 |
$str = str_replace(''', ''', $str);
|
| 1047 | 408 |
$str = preg_replace('/&#([0-9]+);/e', "dechex('$1')", $str);
|
| 1048 | 409 |
// maybe there are some > < ' " & left, replace them too |
| ... | ... | |
| 1057 | 418 |
* |
| 1058 | 419 |
* Will replace all numeric and named entities except |
| 1059 | 420 |
* > < ' " ' |
| 1060 |
* In case of error the returned string is unchanged, and a message is emitted. |
|
| 1061 |
* Supported charsets are: |
|
| 1062 |
* direct: iso_8859_1 iso_8859_2 iso_8859_3 iso_8859_4 iso_8859_5 |
|
| 1063 |
* iso_8859_6 iso_8859_7 iso_8859_8 iso_8859_9 iso_8859_10 iso_8859_11 |
|
| 1064 |
* mb_convert_encoding: all wb charsets (except those from 'direct'); but not GB2312 |
|
| 1065 |
* iconv: all wb charsets (except those from 'direct') |
|
| 1066 |
* |
|
| 1067 |
* @param string A string in DEFAULT_CHARSET encoding |
|
| 1068 |
* @return string A string in $charset_out encoding with numeric and named entities replaced. |
|
| 1069 |
* The string is unchanged in case of error. |
|
| 421 |
* @author thorn |
|
| 1070 | 422 |
*/ |
| 1071 | 423 |
function entities_to_umlauts2($string, $charset_out=DEFAULT_CHARSET) {
|
| 1072 |
$string = charset_to_utf8($string, DEFAULT_CHARSET); |
|
| 1073 |
//if(utf8_check($string)) // this is to much time-consuming
|
|
| 424 |
$string = charset_to_utf8($string, DEFAULT_CHARSET, true);
|
|
| 425 |
//if(utf8_check($string)) // this check is to much time-consuming (this may fail only if AddDefaultCharset is set)
|
|
| 1074 | 426 |
$string = utf8_to_charset($string, $charset_out); |
| 1075 | 427 |
return ($string); |
| 1076 | 428 |
} |
| ... | ... | |
| 1079 | 431 |
* Convert a string from mixed html-entities/umlauts to pure ASCII with HTML-entities |
| 1080 | 432 |
* |
| 1081 | 433 |
* Will convert a string in $charset_in encoding to a pure ASCII string with HTML-entities. |
| 1082 |
* In case of error the returned string is unchanged, and a message is emitted. |
|
| 1083 |
* Supported charsets are: |
|
| 1084 |
* direct: iso_8859_1 iso_8859_2 iso_8859_3 iso_8859_4 iso_8859_5 |
|
| 1085 |
* iso_8859_6 iso_8859_7 iso_8859_8 iso_8859_9 iso_8859_10 iso_8859_11 |
|
| 1086 |
* mb_convert_encoding: all wb charsets (except those from 'direct'); but not GB2312 |
|
| 1087 |
* iconv: all wb charsets (except those from 'direct') |
|
| 1088 |
* |
|
| 1089 |
* @param string A string in $charset_in encoding |
|
| 1090 |
* @return string A string in ASCII encoding with numeric and named entities. |
|
| 1091 |
* The string is unchanged in case of error. |
|
| 434 |
* @author thorn |
|
| 1092 | 435 |
*/ |
| 1093 | 436 |
function umlauts_to_entities2($string, $charset_in=DEFAULT_CHARSET) {
|
| 1094 |
$string = charset_to_utf8($string, $charset_in); |
|
| 1095 |
//if(utf8_check($string)) // this is to much time-consuming
|
|
| 1096 |
$string = utf8_umlauts_to_entities($string);
|
|
| 437 |
$string = charset_to_utf8($string, $charset_in, false);
|
|
| 438 |
//if(utf8_check($string)) // this check is to much time-consuming (this may fail only if AddDefaultCharset is set)
|
|
| 439 |
$string = utf8_fast_umlauts_to_entities($string, false);
|
|
| 1097 | 440 |
return($string); |
| 1098 | 441 |
} |
| 1099 | 442 |
|
Also available in: Unified diff
search: great speed-up with large pages - requires PHP >= 4.3.3; small speed-up for PHP < 4.3.3.
FCK-Editor: loads large pages faster