Synopsis
struct              CRTknzr;
                    CRTknzrPriv;
CRTknzr *           cr_tknzr_new                        (CRInput *a_input);
CRTknzr *           cr_tknzr_new_from_uri               (const guchar *a_file_uri,
                                                         enum CREncoding a_enc);
CRTknzr *           cr_tknzr_new_from_buf               (guchar *a_buf,
                                                         gulong a_len,
                                                         enum CREncoding a_enc,
                                                         gboolean a_free_at_destroy);
gboolean            cr_tknzr_unref                      (CRTknzr *a_this);
void                cr_tknzr_ref                        (CRTknzr *a_this);
enum CRStatus       cr_tknzr_read_byte                  (CRTknzr *a_this,
                                                         guchar *a_byte);
enum CRStatus       cr_tknzr_read_char                  (CRTknzr *a_this,
                                                         guint32 *a_char);
enum CRStatus       cr_tknzr_peek_char                  (CRTknzr *a_this,
                                                         guint32 *a_char);
enum CRStatus       cr_tknzr_peek_byte                  (CRTknzr *a_this,
                                                         gulong a_offset,
                                                         guchar *a_byte);
guchar              cr_tknzr_peek_byte2                 (CRTknzr *a_this,
                                                         gulong a_offset,
                                                         gboolean *a_eof);
enum CRStatus       cr_tknzr_set_cur_pos                (CRTknzr *a_this,
                                                         CRInputPos *a_pos);
glong               cr_tknzr_get_nb_bytes_left          (CRTknzr *a_this);
enum CRStatus       cr_tknzr_get_cur_pos                (CRTknzr *a_this,
                                                         CRInputPos *a_pos);
enum CRStatus       cr_tknzr_get_parsing_location       (CRTknzr *a_this,
                                                         CRParsingLocation *a_loc);
enum CRStatus       cr_tknzr_seek_index                 (CRTknzr *a_this,
                                                         enum CRSeekPos a_origin,
                                                         gint a_pos);
enum CRStatus       cr_tknzr_get_cur_byte_addr          (CRTknzr *a_this,
                                                         guchar **a_addr);
enum CRStatus       cr_tknzr_consume_chars              (CRTknzr *a_this,
                                                         guint32 a_char,
                                                         glong *a_nb_char);
enum CRStatus       cr_tknzr_get_next_token             (CRTknzr *a_this,
                                                         CRToken **a_tk);
enum CRStatus       cr_tknzr_unget_token                (CRTknzr *a_this,
                                                         CRToken *a_token);
enum CRStatus       cr_tknzr_parse_token                (CRTknzr *a_this,
                                                         enum CRTokenType a_type,
                                                         enum CRTokenExtraType a_et,
                                                         gpointer a_res,
                                                         gpointer a_extra_res);
enum CRStatus       cr_tknzr_set_input                  (CRTknzr *a_this,
                                                         CRInput *a_input);
enum CRStatus       cr_tknzr_get_input                  (CRTknzr *a_this,
                                                         CRInput **a_input);
void                cr_tknzr_destroy                    (CRTknzr *a_this);
 
Details
struct CRTknzr
struct CRTknzr {
        /*the private data of the tokenizer.*/
        CRTknzrPriv *priv ;
};
 
CRTknzrPriv
typedef struct _CRTknzrPriv CRTknzrPriv;
 
cr_tknzr_new_from_uri ()
CRTknzr *           cr_tknzr_new_from_uri               (const guchar *a_file_uri,
                                                         enum CREncoding a_enc);
 
cr_tknzr_new_from_buf ()
CRTknzr *           cr_tknzr_new_from_buf               (guchar *a_buf,
                                                         gulong a_len,
                                                         enum CREncoding a_enc,
                                                         gboolean a_free_at_destroy);
 
cr_tknzr_ref ()
void                cr_tknzr_ref                        (CRTknzr *a_this);
 
cr_tknzr_read_byte ()
enum CRStatus       cr_tknzr_read_byte                  (CRTknzr *a_this,
                                                         guchar *a_byte);
 
cr_tknzr_read_char ()
enum CRStatus       cr_tknzr_read_char                  (CRTknzr *a_this,
                                                         guint32 *a_char);
 
cr_tknzr_peek_char ()
enum CRStatus       cr_tknzr_peek_char                  (CRTknzr *a_this,
                                                         guint32 *a_char);
 
cr_tknzr_peek_byte ()
enum CRStatus       cr_tknzr_peek_byte                  (CRTknzr *a_this,
                                                         gulong a_offset,
                                                         guchar *a_byte);
 
cr_tknzr_set_cur_pos ()
enum CRStatus       cr_tknzr_set_cur_pos                (CRTknzr *a_this,
                                                         CRInputPos *a_pos);
 
cr_tknzr_get_nb_bytes_left ()
glong               cr_tknzr_get_nb_bytes_left          (CRTknzr *a_this);
 
cr_tknzr_get_cur_pos ()
enum CRStatus       cr_tknzr_get_cur_pos                (CRTknzr *a_this,
                                                         CRInputPos *a_pos);
 
cr_tknzr_get_parsing_location ()
enum CRStatus       cr_tknzr_get_parsing_location       (CRTknzr *a_this,
                                                         CRParsingLocation *a_loc);
 
cr_tknzr_seek_index ()
enum CRStatus       cr_tknzr_seek_index                 (CRTknzr *a_this,
                                                         enum CRSeekPos a_origin,
                                                         gint a_pos);
 
cr_tknzr_get_cur_byte_addr ()
enum CRStatus       cr_tknzr_get_cur_byte_addr          (CRTknzr *a_this,
                                                         guchar **a_addr);
 
cr_tknzr_consume_chars ()
enum CRStatus       cr_tknzr_consume_chars              (CRTknzr *a_this,
                                                         guint32 a_char,
                                                         glong *a_nb_char);
 
cr_tknzr_get_next_token ()
enum CRStatus       cr_tknzr_get_next_token             (CRTknzr *a_this,
                                                         CRToken **a_tk);
 
cr_tknzr_unget_token ()
enum CRStatus       cr_tknzr_unget_token                (CRTknzr *a_this,
                                                         CRToken *a_token);
 
cr_tknzr_parse_token ()
enum CRStatus       cr_tknzr_parse_token                (CRTknzr *a_this,
                                                         enum CRTokenType a_type,
                                                         enum CRTokenExtraType a_et,
                                                         gpointer a_res,
                                                         gpointer a_extra_res);
 
cr_tknzr_set_input ()
enum CRStatus       cr_tknzr_set_input                  (CRTknzr *a_this,
                                                         CRInput *a_input);
 
cr_tknzr_get_input ()
enum CRStatus       cr_tknzr_get_input                  (CRTknzr *a_this,
                                                         CRInput **a_input);
 
cr_tknzr_destroy ()
void                cr_tknzr_destroy                    (CRTknzr *a_this);