* libidu/idfile.h (struct member_file) [mf_index]: Change

type to long.
* libidu/scanners.h (MAX_LEVELS): Remove constant.
(struct token) [tok_hits, tok_name]: Remove members.
[tok_hits_name]: Add member.
(TOKEN_HITS, TOKEN_NAME, OFFSETOF_TOKEN_NAME): Add macros.
* libidu/scanners.c (log_8_member_files): Add variable definition.
(get_token_c, get_token_asm, get_token_text): Use OFFSETOF_TOKEN_NAME.
* libidu/walker.c (mark_member_file_links) [new_index]: Change
variable type to long.
* src/idx.c (scan_member_file): Use TOKEN_NAME.
* src/mkid.c (ceil_log_8) Add function.
(current_hits_signature): Change variable type to (char *).
(main) [log_8_member_files]: Initialize new variable.
[current_hits_signature] Allocate storage.
(scan_member_file): Print member->mf_index as long.
(scan_member_file_1, write_id_file, token_hash_1, token_hash_2,
token_hash_cmp, token_qsort_cmp, summarize, write_hits):
use TOKEN_NAME & TOKEN_HITS.
(init_hits_signature, write_hits): Replace MAX_LEVELS
with log_8_member_files.
* src/xtokid.c (scan_member_file): Use TOKEN_NAME.

* libidu/idfile.h (vectorize_string) [delimiter_class]:
Declare as pointer to const.
* libidu/walker.c (append_strings_to_vector, vectorize_string)
[delimiter_class]: Declare as pointer to const.
(white_space): Declare as array of const.

7 files changed, 61 insertions, 36 deletions

diff --git a/libidu/idfile.h b/libidu/idfile.h
index 19e265b..5f55073 100644
--- a/libidu/idfile.h
+++ b/libidu/idfile.h
@@ -121,7 +121,7 @@ struct member_file
 {
   struct file_link *mf_link;
   struct lang_args const *mf_lang_args;
-  short mf_index;		/* order in ID file */
+  long mf_index;	/* order in ID file */
 };
 
 #if HAVE_LINK
@@ -199,7 +199,7 @@ extern enum separator_style parse_separator_style __P((char const *arg));
 extern void walk_flink __P((struct file_link *flink, struct dynvec *sub_dirs_vec));
 extern int chdir_to_link __P((struct file_link* dir_link));
 void prune_file_names __P((char *str, struct file_link *from_link));
-char **vectorize_string __P((char *string, char *delimiter_class));
+char **vectorize_string __P((char *string, char const *delimiter_class));
 void include_languages __P((char *lang_names));
 void exclude_languages __P((char *lang_names));
 
diff --git a/libidu/scanners.c b/libidu/scanners.c
index 65a002f..a4a0b70 100644
--- a/libidu/scanners.c
+++ b/libidu/scanners.c
@@ -40,6 +40,7 @@ struct obstack lang_args_obstack;
 struct lang_args *lang_args_default = 0;
 struct lang_args *lang_args_list = 0;
 struct obstack tokens_obstack;
+int log_8_member_files = 0;
 
 extern void usage __P((void));
 extern char *program_name;
@@ -491,7 +492,7 @@ get_token_c (FILE *in_FILE, void const *args, int *flags)
   unsigned char *id = id_0;
   int c;
 
-  obstack_blank (&tokens_obstack, offsetof (struct token, tok_name));
+  obstack_blank (&tokens_obstack, OFFSETOF_TOKEN_NAME);
 
 top:
   c = getc (in_FILE);
@@ -876,7 +877,7 @@ get_token_asm (FILE *in_FILE, void const *args, int *flags)
   unsigned char *id = id_0;
   int c;
 
-  obstack_blank (&tokens_obstack, offsetof (struct token, tok_name));
+  obstack_blank (&tokens_obstack, OFFSETOF_TOKEN_NAME);
 
 top:
   c = getc (in_FILE);
@@ -1161,7 +1162,7 @@ get_token_text (FILE *in_FILE, void const *args, int *flags)
   int c;
   unsigned char *id = id_0;
 
-  obstack_blank (&tokens_obstack, offsetof (struct token, tok_name));
+  obstack_blank (&tokens_obstack, OFFSETOF_TOKEN_NAME);
 
 top:
   c = getc (in_FILE);
diff --git a/libidu/scanners.h b/libidu/scanners.h
index ca95310..aa25aac 100644
--- a/libidu/scanners.h
+++ b/libidu/scanners.h
@@ -21,16 +21,20 @@
 
 #include "xobstack.h"
 
-#define MAX_LEVELS 5		/* log_8 of the max # of files: log_8 (32768) == 5 */
+extern int log_8_member_files;	/* log base 8 of the # of files.
+				   e.g., log_8 (32768) == 5 */
 
 struct token
 {
   unsigned short tok_count;
   unsigned char tok_flags;
-  unsigned char tok_hits[MAX_LEVELS];
-  char tok_name[1];
+  unsigned char tok_hits_name[1];
 };
 
+#define TOKEN_HITS(TOKEN) ((TOKEN)->tok_hits_name)
+#define TOKEN_NAME(TOKEN) ((TOKEN)->tok_hits_name + log_8_member_files)
+#define OFFSETOF_TOKEN_NAME (offsetof (struct token, tok_hits_name) + log_8_member_files)
+
 typedef struct token *(*get_token_func_t) __P((FILE *in_FILE, void const *args, int *flags));
 typedef void *(*parse_args_func_t) __P((char **argv, int argc));
 typedef void (*help_me_func_t) __P((void));
diff --git a/libidu/walker.c b/libidu/walker.c
index 80d1b59..63fba09 100644
--- a/libidu/walker.c
+++ b/libidu/walker.c
@@ -46,7 +46,7 @@ struct file_link *make_link_from_dirent __P((struct dirent *dirent, struct file_
 struct file_link *get_link_from_string __P((char const *name, struct file_link *parent));
 struct file_link *make_link_from_string __P((char const *name, struct file_link *parent));
 int lang_wanted __P((char const *lang_name));
-char **append_strings_to_vector __P((char **vector_0, char *string, char *delimiter_class));
+char **append_strings_to_vector __P((char **vector_0, char *string, char const *delimiter_class));
 int vector_length __P((char **vector));
 int string_in_vector __P((char const *string, char **vector));
 static int same_as_dot __P((char const *cwd));
@@ -77,7 +77,7 @@ struct member_file *maybe_get_member_file __P((struct file_link *flink, struct s
 
 static struct file_link *current_dir_link = 0;
 
-static char white_space[] = " \t\r\n\v\f";
+static char const white_space[] = " \t\r\n\v\f";
 
 char* xgetcwd __P((void));
 
@@ -242,7 +242,7 @@ mark_member_file_links (struct idhead *idhp)
 					 0, member_file_qsort_compare);
   struct member_file **end = &members_0[idhp->idh_member_file_table.ht_fill];
   struct member_file **members;
-  int new_index = 0;
+  long new_index = 0;
 
   for (members = members_0; members < end; members++)
     {
@@ -467,7 +467,7 @@ exclude_languages (char *lang_names)
 }
 
 char **
-append_strings_to_vector (char **vector_0, char *string, char *delimiter_class)
+append_strings_to_vector (char **vector_0, char *string, char const *delimiter_class)
 {
   char **vector;
   if (vector_0)
@@ -650,7 +650,7 @@ chdir_to_link (struct file_link *dir_link)
 }
 
 char **
-vectorize_string (char *string, char *delimiter_class)
+vectorize_string (char *string, char const *delimiter_class)
 {
   char **vector_0 = MALLOC (char *, 2 + strlen (string) / 2);
   char **vector = vector_0;
diff --git a/src/idx.c b/src/idx.c
index 60ca5fc..a6fb8c7 100644
--- a/src/idx.c
+++ b/src/idx.c
@@ -106,7 +106,7 @@ main (int argc, char **argv)
 
   if (show_help)
     help_me ();
-  
+
   argc -= optind;
   argv += optind;
 
@@ -159,7 +159,7 @@ scan_member_file (struct member_file const *member)
 
       while ((token = (*get_token) (source_FILE, args, &flags)) != NULL)
 	{
-	  puts (token->tok_name);
+	  puts (TOKEN_NAME (token));
 	  obstack_free (&tokens_obstack, token);
 	}
       close_source_FILE (source_FILE);
diff --git a/src/mkid.c b/src/mkid.c
index 3168c49..b2c0bf7 100644
--- a/src/mkid.c
+++ b/src/mkid.c
@@ -58,6 +58,7 @@ struct summary
 void usage __P((void));
 static void help_me __P((void));
 int main __P((int argc, char **argv));
+int ceil_log_8 __P((unsigned long n));
 void assert_writeable __P((char const *file_name));
 void scan_files __P((struct idhead *idhp));
 void scan_member_file __P((struct member_file const *member));
@@ -101,7 +102,7 @@ int statistics_flag = 0;
 int file_name_count = 0;	/* # of files in database */
 int levels = 0;			/* ceil(log(8)) of file_name_count */
 
-unsigned char current_hits_signature[MAX_LEVELS];
+char *current_hits_signature;
 #define INIT_TOKENS_SIZE(level) (1 << ((level) + 13))
 struct summary *summary_root;
 struct summary *summary_leaf;
@@ -280,6 +281,8 @@ main (int argc, char **argv)
   heap_after_walk = (char const *) sbrk (0);
 
   mark_member_file_links (&idh);
+  log_8_member_files = ceil_log_8 (idh.idh_member_file_table.ht_fill);
+  current_hits_signature = MALLOC (char, log_8_member_files);
   if (idh.idh_member_file_table.ht_fill)
     {
       scan_files (&idh);
@@ -295,9 +298,26 @@ main (int argc, char **argv)
     }
   else
     error (0, 0, "nothing to do");
+
   exit (0);
 }
 
+/* Return the integer ceiling of the base-8 logarithm of N.  */
+
+int
+ceil_log_8 (unsigned long n)
+{
+  int log_8 = 0;
+
+  n--;
+  while (n)
+    {
+      log_8++;
+      n >>= 3;
+    }
+  return log_8;
+}
+
 void
 assert_writeable (char const *file_name)
 {
@@ -374,7 +394,7 @@ scan_member_file (struct member_file const *member)
       if (verbose_flag)
 	{
 	  maybe_relative_file_name (file_name, flink, cw_dlink);
-	  printf ("%d: %s: %s", member->mf_index, lang->lg_name, file_name);
+	  printf ("%ld: %s: %s", member->mf_index, lang->lg_name, file_name);
 	  fflush (stdout);
 	}
       scan_member_file_1 (get_token, lang_args->la_args_digested, source_FILE);
@@ -397,7 +417,7 @@ scan_member_file_1 (get_token_func_t get_token, void const *args, FILE *source_F
 
   while ((token = (*get_token) (source_FILE, args, &flags)) != NULL)
     {
-      if (*token->tok_name == '\0') {
+      if (*TOKEN_NAME (token) == '\0') {
 	obstack_free (&tokens_obstack, token);
 	continue;
       }
@@ -406,7 +426,7 @@ scan_member_file_1 (get_token_func_t get_token, void const *args, FILE *source_F
 	{
 	  token->tok_flags = flags;
 	  token->tok_count = 1;
-	  memset (token->tok_hits, 0, sizeof (token->tok_hits));
+	  memset (TOKEN_HITS (token), 0, log_8_member_files);
 	  sign_token (token);
 	  if (verbose_flag)
 	    {
@@ -422,7 +442,7 @@ scan_member_file_1 (get_token_func_t get_token, void const *args, FILE *source_F
 	  token->tok_flags |= flags;
 	  if (token->tok_count < USHRT_MAX)
 	    token->tok_count++;
-	  if (!(token->tok_hits[0] & current_hits_signature[0]))
+	  if (!(TOKEN_HITS (token)[0] & current_hits_signature[0]))
 	    {
 	      sign_token (token);
 	      if (verbose_flag)
@@ -521,7 +541,7 @@ write_id_file (struct idhead *idhp)
       if (token->tok_flags & TOK_COMMENT)
 	comment_tokens++;
 
-      fputs (token->tok_name, idhp->idh_FILE);
+      fputs (TOKEN_NAME (token), idhp->idh_FILE);
       putc ('\0', idhp->idh_FILE);
       if (token->tok_count > 0xff)
 	token->tok_flags |= TOK_SHORT_COUNT;
@@ -530,10 +550,10 @@ write_id_file (struct idhead *idhp)
       if (token->tok_flags & TOK_SHORT_COUNT)
 	putc (token->tok_count >> 8, idhp->idh_FILE);
 
-      vec_size = count_vec_size (summary_root, token->tok_hits + levels);
-      buf_size = count_buf_size (summary_root, token->tok_hits + levels);
+      vec_size = count_vec_size (summary_root, TOKEN_HITS (token) + levels);
+      buf_size = count_buf_size (summary_root, TOKEN_HITS (token) + levels);
       hits_length += buf_size;
-      tok_size = strlen (token->tok_name) + 1;
+      tok_size = strlen (TOKEN_NAME (token)) + 1;
       tokens_length += tok_size;
       buf_size += tok_size + sizeof (token->tok_flags) + sizeof (token->tok_count) + 2;
       if (buf_size > max_buf_size)
@@ -541,7 +561,7 @@ write_id_file (struct idhead *idhp)
       if (vec_size > max_vec_size)
 	max_vec_size = vec_size;
 
-      write_hits (idhp->idh_FILE, summary_root, token->tok_hits + levels);
+      write_hits (idhp->idh_FILE, summary_root, TOKEN_HITS (token) + levels);
       putc ('\0', idhp->idh_FILE);
       putc ('\0', idhp->idh_FILE);
     }
@@ -559,27 +579,27 @@ write_id_file (struct idhead *idhp)
 unsigned long
 token_hash_1 (void const *key)
 {
-  return_STRING_HASH_1 (((struct token const *) key)->tok_name);
+  return_STRING_HASH_1 (TOKEN_NAME ((struct token const *) key));
 }
 
 unsigned long
 token_hash_2 (void const *key)
 {
-  return_STRING_HASH_2 (((struct token const *) key)->tok_name);
+  return_STRING_HASH_2 (TOKEN_NAME ((struct token const *) key));
 }
 
 int
 token_hash_cmp (void const *x, void const *y)
 {
-  return_STRING_COMPARE (((struct token const *) x)->tok_name,
-			 ((struct token const *) y)->tok_name);
+  return_STRING_COMPARE (TOKEN_NAME ((struct token const *) x),
+			 TOKEN_NAME ((struct token const *) y));
 }
 
 int
 token_qsort_cmp (void const *x, void const *y)
 {
-  return_STRING_COMPARE ((*(struct token const *const *) x)->tok_name,
-			 (*(struct token const *const *) y)->tok_name);
+  return_STRING_COMPARE (TOKEN_NAME (*(struct token const *const *) x),
+			 TOKEN_NAME (*(struct token const *const *) y));
 }
 
 
@@ -598,7 +618,7 @@ void
 init_hits_signature (int i)
 {
   unsigned char *hits = current_hits_signature;
-  unsigned char const *end = &current_hits_signature[MAX_LEVELS];
+  unsigned char const *end = current_hits_signature + log_8_member_files;
   while (hits < end)
     {
       *hits = 1 << (i & 7);
@@ -641,7 +661,7 @@ summarize (void)
       summary->sum_hits = hits;
       while (count--)
 	{
-	  unsigned char *hit = &(*tokens++)->tok_hits[level];
+	  unsigned char *hit = TOKEN_HITS (*tokens++) + level;
 	  *hits++ = *hit;
 	  *hit = 0;
 	}
@@ -788,9 +808,9 @@ write_hits (FILE *fp, struct summary *summary, unsigned char const *tail_hits)
 void
 sign_token (struct token *token)
 {
-  unsigned char *tok_hits = token->tok_hits;
+  unsigned char *tok_hits = TOKEN_HITS (token);
   unsigned char *hits_sig = current_hits_signature;
-  unsigned char *end = &current_hits_signature[MAX_LEVELS];
+  unsigned char *end = current_hits_signature + log_8_member_files;
   struct summary *summary = summary_leaf;
 
   while (summary)
diff --git a/src/xtokid.c b/src/xtokid.c
index 5b1352c..80f146c 100644
--- a/src/xtokid.c
+++ b/src/xtokid.c
@@ -206,7 +206,7 @@ scan_member_file (struct member_file const *member)
 
       while ((token = (*get_token) (source_FILE, args, &flags)) != NULL)
 	{
-	  puts (token->tok_name);
+	  puts (TOKEN_NAME (token));
 	  obstack_free (&tokens_obstack, token);
 	}
       fclose (source_FILE);