From 6291ad9865afff38f6cb96906ff8afade950b868 Mon Sep 17 00:00:00 2001
From: midipix <writeonce@midipix.org>
Date: May 15 2024 04:57:14 +0000
Subject: library api: added token scanner definitions and structures.


---

diff --git a/include/treebnf/treebnf.h b/include/treebnf/treebnf.h
index c10f115..1a8db27 100644
--- a/include/treebnf/treebnf.h
+++ b/include/treebnf/treebnf.h
@@ -24,6 +24,11 @@ extern "C" {
 #define TBNF_USAGE			0x01
 #define TBNF_ERROR			0x02
 
+/* token scanner state ops */
+#define TBNF_STATE_POP                  (-1)
+#define TBNF_STATE_KEEP                 ( 0)
+#define TBNF_STATE_PUSH                 (+1)
+
 /* driver flags */
 #define TBNF_DRIVER_XFLAG(x)		((uint64_t)(x) << 32)
 
@@ -51,6 +56,39 @@ enum tbnf_custom_error {
 	TBNF_ERR_IMAGE_SIZE_ZERO,
 };
 
+/* token api structures */
+struct tbnf_token;
+struct tbnf_scan_tbl;
+struct tbnf_scan_ctx;
+
+struct tbnf_token {
+	int                     tok_type;
+	int                     tok_len;
+	int                     tok_off;
+	int                     tok_any;
+};
+
+struct tbnf_scan_tbl {
+	int                     (*tok_scan_fn)(const struct tbnf_scan_ctx *);
+	int                     tok_state_op;
+	int                     tok_state_next;
+};
+
+struct tbnf_scan_ctx {
+	const char *            tok_scan_base;
+	const char *            tok_scan_mark;
+	const char *            tok_scan_cap;
+	int                     tok_scan_type;
+	int                     tok_scan_state;
+	int                     tok_scan_nents;
+	int                     tok_scan_ntbls;
+	struct tbnf_scan_tbl ** tok_scan_tbls;
+};
+
+
+
+
+
 struct tbnf_raw_input {
 	void *          map_addr;
 	size_t          map_size;