Copyright (c) 2019 JoungKyun.Kim http://oops.org All rights reserved.
This program is under MPL 1.1 or LGPL 2.1
libchardet is based on Mozilla Universal Charset Detector library and, detects the character set used to encode data.
Original code was writed by Netscape Communications Corporation, Techniques used by universalchardet are described at <http://www-archive.mozilla.org/projects/intl/UniversalCharsetDetection.html>.
libchardet see also John Gardiner Myers's Encode-Detect-1.01 perl module, and added C wrapping API, and library build environment with libtool.
From 1.0.5, libchardet was reflected single-byte charset detection confidence algorithm of uchardet and new language models. (Arabic, Danish, Esperanto, German, Spanish, Turkish, Vietnamese)
From 1.0.6, bom members have been added to the DetectObj structure. The value of the bom member is 1, which means that it has been detected as a BOM. Support for bom member can be determined by the existence of the CHARDET_BOM_CHECK constant. See example below.
See also INSTALL document
See also test directory of source code
#include <chardet.h>
int main (void) {
DetectObj *obj;
char * str = "안녕하세요";
if ( (obj = detect_obj_init ()) == NULL ) {
fprintf (stderr, "Memory Allocation failed\n");
return CHARDET_MEM_ALLOCATED_FAIL;
}
#ifndef CHARDET_BINARY_SAFE
// before 1.0.5. This API is deprecated on 1.0.5
switch (detect (str, &obj))
#else
// from 1.0.5
switch (detect_r (str, strlen (str), &obj))
#endif
{
case CHARDET_OUT_OF_MEMORY :
fprintf (stderr, "On handle processing, occured out of memory\n");
detect_obj_free (&obj);
return CHARDET_OUT_OF_MEMORY;
case CHARDET_NULL_OBJECT :
fprintf (stderr,
"2st argument of chardet() is must memory allocation "
"with detect_obj_init API\n");
return CHARDET_NULL_OBJECT;
}
#ifndef CHARDET_BOM_CHECK
printf ("encoding: %s, confidence: %f\n", obj->encoding, obj->confidence);
#else
// from 1.0.6 support return whether exists BOM
printf (
"encoding: %s, confidence: %f, exist BOM: %d\n",
obj->encoding, obj->confidence, obj->bom
);
#endif
detect_obj_free (&obj);
return 0;
}or looping code
#include <chardet.h>
int main (void) {
Detect * d;
DetectObj * obj;
char * str = "안녕하세요";
if ( (d = detect_init ()) == NULL ) {
fprintf (stderr, "chardet handle initialize failed\n");
return CHARDET_MEM_ALLOCATED_FAIL;
}
while ( 1 ) {
detect_reset (&d);
if ( (obj = detect_obj_init ()) == NULL ) {
fprintf (stderr, "Memory Allocation failed\n");
return CHARDET_MEM_ALLOCATED_FAIL;
}
#ifndef CHARDET_BINARY_SAFE
// before 1.0.5. This API is deprecated on 1.0.5
switch (detect_handledata (&d, str,, &obj))
#else
// from 1.0.5
switch (detect_handledata_r (&d, str, strlen (str), &obj))
#endif
{
case CHARDET_OUT_OF_MEMORY :
fprintf (stderr, "On handle processing, occured out of memory\n");
detect_obj_free (&obj);
return CHARDET_OUT_OF_MEMORY;
case CHARDET_NULL_OBJECT :
fprintf (stderr,
"2st argument of chardet() is must memory allocation "
"with detect_obj_init API\n");
return CHARDET_NULL_OBJECT;
}
#ifndef CHARDET_BOM_CHECK
printf ("encoding: %s, confidence: %f\n", obj->encoding, obj->confidence);
#else
// from 1.0.6 support return whether exists BOM
printf (
"encoding: %s, confidence: %f, exist BOM: %d\n",
obj->encoding, obj->confidence, obj->bom
);
#endif
detect_obj_free (&obj);
if ( 1 )
break;
}
detect_destroy (&d);
return 0;
}- PHP Extension - https://github.com/OOPS-ORG-PHP/mod_chardet
- PYTHON C Binding - https://github.com/Joungkyun/python-chardet
- PERL - http://search.cpan.org/~jgmyers/Encode-Detect-1.01/Detect.pm
