Nmm_ocaml : Nmm_lexer.token

let rec token (lexbuf : Sedlexing.lexbuf) : Nmm_parser.token=
        match verbatim.contents, display.contents with
        |false, false -> (
                match%sedlex lexbuf with
                |esc_char                        ->      ESC_CHAR (get_esc_char (lexeme lexbuf))
                |preamble_colon                  ->      PREAMBLE_COLON
                |title_colon                     ->      TITLE_COLON
                |author_colon                    ->      AUTHOR_COLON
                |date_colon                      ->      DATE_COLON
                |abstract_colon                  ->      ABSTRACT_COLON
                |ch_tag_or_id_nl                 ->      CH_TAG_OR_ID_NL (String.trim (lexeme lexbuf))
                |nte_ref                         ->      NTE_REF (lexeme lexbuf, nte_count ())
                |c_ref                           ->      C_REF (lexeme lexbuf)
                |section_nl                      ->      SECTION_NL
                |section_spaces_tag_or_id_nl     ->      SECTION_SPACES_TAG_OR_ID_NL (get_tag_or_id (lexeme lexbuf))
                |pilcrow_nl                      ->      PILCROW_NL
                |pilcrow_spaces_tag_or_id_nl     ->      PILCROW_SPACES_TAG_OR_ID_NL (get_tag_or_id (lexeme lexbuf))
                |pilcrow_spaces_rpt_spaces_id_nl ->     PILCROW_SPACES_RPT_SPACES_ID_NL (get_id (lexeme lexbuf))
                |section_refs_nls                ->      SECTION_REFS_NLS
                |pilcrow_refs_nls                ->      PILCROW_REFS_NLS
                |tab                             ->      TAB 
                |dash_tab                        ->      DASH_TAB
                |star_tab_id                     ->      STAR_TAB_ID (lexeme lexbuf)
                |dsp_auto_tab                    ->      set_display_and_return_token DSP_AUTO_TAB 
                |dsp_custom_tab                  ->      set_display_and_return_token (DSP_CUSTOM_TAB (get_label (lexeme lexbuf)))
                |itm_auto_tab                    ->      ITM_AUTO_TAB
                |itm_custom_tab                  ->      ITM_CUSTOM_TAB (get_label (lexeme lexbuf))
                |itm_auto_tab_id                 ->      ITM_AUTO_TAB_ID (lexeme lexbuf)
                |itm_custom_tab_id               ->      ITM_CUSTOM_TAB_ID (lexeme lexbuf)
                |nl                              ->      let _ : unit = skip_newlines lexbuf in NL
                |nl_tab                          ->      NL_TAB
                |nl_tab_tab                      ->      NL_TAB_TAB
                |nl_tab_tab_tab                  ->      NL_TAB_TAB_TAB
                |star                            ->      STAR
                |lbr                             ->      LBR
                |rbr                             ->      RBR
                |colon                           ->      COLON
                |section                         ->      SECTION
                |pilcrow                         ->      PILCROW
                |nte_lbr                         ->      NTE_LBR (nte_count ())
                |txt                             ->      TXT (lexeme lexbuf)
                |start_vrb                       ->      set_verbatim_and_return_token START_VRB
                |eof                             ->      if end_of_file.contents then EOF else set_end_of_file_and_return_token NL
                |_ -> raise (ERROR ("unexpected string on line " ^ (line_of_lexbuf lexbuf) ^ ": \"" ^ (lexeme lexbuf) ^ "\""))
        )

        |true, _ -> (
                match%sedlex lexbuf with
                |end_vrb                        ->      reset_verbatim_and_return_token END_VRB
                |tab_end_vrb                    ->      reset_verbatim_and_return_token TAB_END_VRB
                |tab_tab_end_vrb                ->      reset_verbatim_and_return_token TAB_TAB_END_VRB
                |tab_tab_tab_end_vrb            ->      reset_verbatim_and_return_token TAB_TAB_TAB_END_VRB
                |vrb_line                       ->      VRB_LINE (remove_nls (lexeme lexbuf))
                |nl                             ->      VRB_LINE_EMPTY
                |tab                            ->      TAB
                |_ -> raise (ERROR ("unexpected string on line " ^ (line_of_lexbuf lexbuf) ^ ": \"" ^ (lexeme lexbuf) ^ "\""))
        )

        |_, true -> (
                match%sedlex lexbuf with
                |esc_char                       ->      ESC_CHAR (get_esc_char (lexeme lexbuf))
                |star                           ->      STAR
                |lbr                            ->      LBR
                |rbr                            ->      RBR
                |colon                          ->      COLON
                |section                        ->      SECTION
                |pilcrow                        ->      PILCROW
                |nte_ref                        ->      NTE_REF (lexeme lexbuf, nte_count ())
                |c_ref                          ->      C_REF (lexeme lexbuf)
                |nte_lbr                        ->      NTE_LBR (nte_count ())
                |txt                            ->      TXT (lexeme lexbuf)
                |tab                            ->      TAB 
                |dsp_id                         ->      DSP_ID (lexeme lexbuf)
                |nl                             ->      reset_display_and_return_token NL
                |nl_tab                         ->      reset_display_and_return_token NL_TAB
                |nl_tab_tab                     ->      reset_display_and_return_token NL_TAB_TAB
                |nl_tab_tab_tab                 ->      reset_display_and_return_token NL_TAB_TAB_TAB
                |_ -> raise (ERROR ("unexpected string on line " ^ (line_of_lexbuf lexbuf) ^ ": \"" ^ (lexeme lexbuf) ^ "\""))
        )


and skip_newlines (lexbuf : Sedlexing.lexbuf) : unit =
	match%sedlex lexbuf with
	|nl_not_nl -> Sedlexing.rollback lexbuf
	|nl -> skip_newlines lexbuf
	|_ -> ()