* Version 3.2c released.

* libidu/id-file.h (walker_verbose_flag): Add extern variable decl.
* libidu/scanners.c (SCAN_CPP_DIRECTIVE): Add macro.
(get_token_c, get_token_asm): Call SCAN_CPP_DIRECTIVE.
* libidu/walker.c (walker_verbose_flag): Add variable defn.
(print_member_file): Add function.
(maybe_get_member_file): Call print_member_file in verbose mode.
* src/mkid.c: Add coments.  (ceil_log_2): Add function.
(main): Handle 'V' option for extra verbosity.
(scan_files): Use MAX (1M, n*lg(n)*64) to estimate initial size
of token hash table.

4 files changed, 151 insertions, 104 deletions

diff --git a/libidu/idfile.h b/libidu/idfile.h
index 5f55073..fdfb704 100644
--- a/libidu/idfile.h
+++ b/libidu/idfile.h
@@ -219,6 +219,8 @@ extern struct member_file *find_member_file __P((struct file_link const *flink))
 
 extern struct idhead idh;
 
+extern int walker_verbose_flag;
+
 #define DEFAULT_ID_FILE_NAME "ID"
 
 #endif /* not _idfile_h_ */
diff --git a/libidu/scanners.c b/libidu/scanners.c
index a4a0b70..ea533fe 100644
--- a/libidu/scanners.c
+++ b/libidu/scanners.c
@@ -480,6 +480,81 @@ parse_args_c (char **argv, int argc)
 
 static unsigned char id_0[1<<020];
 
+#define SCAN_CPP_DIRECTIVE						\
+  do									\
+    {									\
+      c = getc (in_FILE);						\
+      while (ISBORING (c))						\
+	c = getc (in_FILE);						\
+      if (!ISID1ST (c))							\
+	goto next;							\
+      id = id_0;							\
+      *id++ = c;							\
+      while (ISIDREST (c = getc (in_FILE)))				\
+	*id++ = c;							\
+      *id = '\0';							\
+      if (strequ (id_0, "include"))					\
+	{								\
+	  while (c == ' ' || c == '\t')					\
+	    c = getc (in_FILE);						\
+	  if (c == '\n')						\
+	    {								\
+	      new_line = 1;						\
+	      goto top;							\
+	    }								\
+	  id = id_0;							\
+	  if (c == '"')							\
+	    {								\
+	      c = getc (in_FILE);					\
+	      while (c != '\n' && c != EOF && c != '"')			\
+		{							\
+		  *id++ = c;						\
+		  c = getc (in_FILE);					\
+		}							\
+	      *flags = TOK_STRING;					\
+	    }								\
+	  else if (c == '<')						\
+	    {								\
+	      c = getc (in_FILE);					\
+	      while (c != '\n' && c != EOF && c != '>')			\
+		{							\
+		  *id++ = c;						\
+		  c = getc (in_FILE);					\
+		}							\
+	      *flags = TOK_STRING;					\
+	    }								\
+	  else if (ISID1ST (c))						\
+	    {								\
+	      *id++ = c;						\
+	      while (ISIDREST (c = getc (in_FILE)))			\
+		*id++ = c;						\
+	      *flags = TOK_NAME;					\
+	    }								\
+	  else								\
+	    {								\
+	      while (c != '\n' && c != EOF)				\
+		c = getc (in_FILE);					\
+	      new_line = 1;						\
+	      goto top;							\
+	    }								\
+	  while (c != '\n' && c != EOF)					\
+	    c = getc (in_FILE);						\
+	  new_line = 1;							\
+	  obstack_grow0 (&tokens_obstack, id_0, id - id_0);		\
+	  return (struct token *) obstack_finish (&tokens_obstack);	\
+	}								\
+      if (strnequ (id_0, "if", 2)					\
+	  || strequ (id_0, "define")					\
+	  || strequ (id_0, "elif")	/* ansi C */			\
+	  || strequ (id_0, "undef"))					\
+	goto next;							\
+      while ((c != '\n') && (c != EOF))					\
+	c = getc (in_FILE);						\
+      new_line = 1;							\
+      goto top;								\
+  } while (0)
+
+
 /* Grab the next identifier from the C source file.  This state
    machine is built for speed, not elegance.  */
 
@@ -501,75 +576,7 @@ top:
       new_line = 0;
       if (c != '#')
 	goto next;
-      c = getc (in_FILE);
-      while (ISBORING (c))
-	c = getc (in_FILE);
-      if (!ISID1ST (c))
-	goto next;
-      id = id_0;
-      *id++ = c;
-      while (ISIDREST (c = getc (in_FILE)))
-	*id++ = c;
-      *id = '\0';
-      if (strequ (id_0, "include"))
-	{
-	  while (c == ' ' || c == '\t')
-	    c = getc (in_FILE);
-	  if (c == '\n')
-	    {
-	      new_line = 1;
-	      goto top;
-	    }
-	  id = id_0;
-	  if (c == '"')
-	    {
-	      c = getc (in_FILE);
-	      while (c != '\n' && c != EOF && c != '"')
-		{
-		  *id++ = c;
-		  c = getc (in_FILE);
-		}
-	      *flags = TOK_STRING;
-	    }
-	  else if (c == '<')
-	    {
-	      c = getc (in_FILE);
-	      while (c != '\n' && c != EOF && c != '>')
-		{
-		  *id++ = c;
-		  c = getc (in_FILE);
-		}
-	      *flags = TOK_STRING;
-	    }
-	  else if (ISID1ST (c))
-	    {
-	      *id++ = c;
-	      while (ISIDREST (c = getc (in_FILE)))
-		*id++ = c;
-	      *flags = TOK_NAME;
-	    }
-	  else
-	    {
-	      while (c != '\n' && c != EOF)
-		c = getc (in_FILE);
-	      new_line = 1;
-	      goto top;
-	    }
-	  while (c != '\n' && c != EOF)
-	    c = getc (in_FILE);
-	  new_line = 1;
-	  obstack_grow0 (&tokens_obstack, id_0, id - id_0);
-	  return (struct token *) obstack_finish (&tokens_obstack);
-	}
-      if (strnequ (id_0, "if", 2)
-	  || strequ (id_0, "define")
-	  || strequ (id_0, "elif")	/* ansi C */
-	  || strequ (id_0, "undef"))
-	goto next;
-      while ((c != '\n') && (c != EOF))
-	c = getc (in_FILE);
-      new_line = 1;
-      goto top;
+      SCAN_CPP_DIRECTIVE;
     }
 
 next:
@@ -886,35 +893,7 @@ top:
       new_line = 0;
       if (c != '#')
 	goto next;
-      while (ISBORING (c))
-	c = getc (in_FILE);
-      if (!ISID1ST (c))
-	goto next;
-      id = id_0;
-      *id++ = c;
-      while (ISIDREST (c = getc (in_FILE)))
-	*id++ = c;
-      *id = '\0';
-      if (strequ (id_0, "include"))
-	{
-	  while (c != '"' && c != '<')
-	    c = getc (in_FILE);
-	  id = id_0;
-	  *id++ = c = getc (in_FILE);
-	  while ((c = getc (in_FILE)) != '"' && c != '>')
-	    *id++ = c;
-	  *flags = TOK_STRING;
-	  obstack_grow0 (&tokens_obstack, id_0, id - id_0);
-	  return (struct token *) obstack_finish (&tokens_obstack);
-	}
-      if (strnequ (id_0, "if", 2)
-	  || strequ (id_0, "define")
-	  || strequ (id_0, "undef"))
-	goto next;
-      while (c != '\n')
-	c = getc (in_FILE);
-      new_line = 1;
-      goto top;
+      SCAN_CPP_DIRECTIVE;
     }
 
 next:
diff --git a/libidu/walker.c b/libidu/walker.c
index 63fba09..e862c73 100644
--- a/libidu/walker.c
+++ b/libidu/walker.c
@@ -35,9 +35,12 @@
 #include "pathmax.h"
 #include "xalloca.h"
 
+int walker_verbose_flag = 0;
+
 int walk_dir __P((struct file_link *dir_link));
 struct member_file *get_member_file __P((struct file_link *flink));
 struct lang_args *get_lang_args __P((struct file_link const *flink));
+void print_member_file (struct member_file *member);
 int walk_sub_dirs __P((struct dynvec *sub_dirs_vec));
 void reparent_children __P((struct file_link *dlink, struct file_link *slink));
 int classify_link __P((struct file_link *flink, struct stat *stp));
@@ -302,6 +305,8 @@ maybe_get_member_file (struct file_link *flink, struct stat *stp)
 	  alias_member->mf_link->fl_flags &= ~FL_MEMBER;
 	}
     }
+  if (member && walker_verbose_flag)
+    print_member_file (member);
   return member;
 }
 
@@ -433,6 +438,15 @@ get_lang_args (struct file_link const *flink)
 	  ? lang_args_default : 0);
 }
 
+void
+print_member_file (struct member_file *member)
+{
+  char *file_name = ALLOCA (char, PATH_MAX);
+  absolute_file_name (file_name, member->mf_link);
+  printf ("%ld: %s: %s\n", idh.idh_member_file_table.ht_fill - 1,
+	  member->mf_lang_args->la_language->lg_name, file_name);
+}
+
 
 /****************************************************************************/
 
diff --git a/src/mkid.c b/src/mkid.c
index b2c0bf7..b35c1f3 100644
--- a/src/mkid.c
+++ b/src/mkid.c
@@ -59,6 +59,7 @@ void usage __P((void));
 static void help_me __P((void));
 int main __P((int argc, char **argv));
 int ceil_log_8 __P((unsigned long n));
+int ceil_log_2 __P((unsigned long n));
 void assert_writeable __P((char const *file_name));
 void scan_files __P((struct idhead *idhp));
 void scan_member_file __P((struct member_file const *member));
@@ -193,7 +194,7 @@ main (int argc, char **argv)
 
   for (;;)
     {
-      int optc = getopt_long (argc, argv, "o:f:i:x:l:m:d:p:vs",
+      int optc = getopt_long (argc, argv, "o:f:i:x:l:m:d:p:vVs",
 			      long_options, (int *) 0);
       if (optc < 0)
 	break;
@@ -233,11 +234,10 @@ main (int argc, char **argv)
 	  prune_file_names (optarg, cw_dlink);
 	  break;
 
+	case 'V':
+	  walker_verbose_flag = 1;
 	case 'v':
 	  verbose_flag = 1;
-	  statistics_flag = 1;
-	  break;
-
 	case 's':
 	  statistics_flag = 1;
 	  break;
@@ -258,6 +258,8 @@ main (int argc, char **argv)
 
   argc -= optind;
   argv += optind;
+
+  /* If no file or directory options exist, walk the current directory.  */
   if (argc == 0)
     {
       static char dot[] = ".";
@@ -272,6 +274,7 @@ main (int argc, char **argv)
     cw_dlink = init_walker (&idh);
   parse_language_map (lang_map_file_name);
 
+  /* Walk the file and directory names given on the command line.  */
   while (argc--)
     {
       struct file_link *flink = parse_file_name (*argv++, cw_dlink);
@@ -283,6 +286,8 @@ main (int argc, char **argv)
   mark_member_file_links (&idh);
   log_8_member_files = ceil_log_8 (idh.idh_member_file_table.ht_fill);
   current_hits_signature = MALLOC (char, log_8_member_files);
+
+  /* If scannable files were given, then scan them.  */
   if (idh.idh_member_file_table.ht_fill)
     {
       scan_files (&idh);
@@ -318,6 +323,26 @@ ceil_log_8 (unsigned long n)
   return log_8;
 }
 
+/* Return the integer ceiling of the base-8 logarithm of N.  */
+
+int
+ceil_log_2 (unsigned long n)
+{
+  int log_2 = 0;
+
+  n--;
+  while (n)
+    {
+      log_2++;
+      n >>= 1;
+    }
+  return log_2;
+}
+
+/* Ensure that FILE_NAME can be written.  If it exists, make sure it's
+   writable.  If it doesn't exist, make sure it can be created.  Exit
+   with a diagnostic if this is not so.  */
+
 void
 assert_writeable (char const *file_name)
 {
@@ -337,6 +362,9 @@ assert_writeable (char const *file_name)
     }
 }
 
+/* Iterate over all eligible files (the members of the set of scannable files).
+   Create a tree8 to store the set of files where a token occurs.  */
+
 void
 scan_files (struct idhead *idhp)
 {
@@ -345,9 +373,19 @@ scan_files (struct idhead *idhp)
 					 0, member_file_qsort_compare);
   struct member_file **end = &members_0[idhp->idh_member_file_table.ht_fill];
   struct member_file **members = members_0;
+  int n = idhp->idh_member_file_table.ht_fill;
+
+  n = n * ceil_log_2 (n) * 16;
+  if (n == 0)
+    n = 1024;
+  else if (n > 1024*1024)
+    n = 1024*1024;
 
-  hash_init (&token_table, idhp->idh_member_file_table.ht_fill * 64,
-	     token_hash_1, token_hash_2, token_hash_cmp);
+  hash_init (&token_table, n, token_hash_1, token_hash_2, token_hash_cmp);
+  if (verbose_flag)
+    printf ("member files=%ld, token table slots=%ld\n",
+	    idhp->idh_member_file_table.ht_fill,
+	    token_table.ht_size);
   init_hits_signature (0);
   init_summary ();
   obstack_init (&tokens_obstack);
@@ -366,6 +404,8 @@ scan_files (struct idhead *idhp)
   free (members_0);
 }
 
+/* Open a file and scan it.  */
+
 void
 scan_member_file (struct member_file const *member)
 {
@@ -406,6 +446,9 @@ scan_member_file (struct member_file const *member)
     error (0, errno, _("can't open `%s'"), flink->fl_name);
 }
 
+/* Iterate over all tokens in the file, and merge the file's tree8
+   signature into the token table entry.  */
+
 void
 scan_member_file_1 (get_token_func_t get_token, void const *args, FILE *source_FILE)
 {
@@ -486,6 +529,7 @@ report_statistics (void)
    (This would be a common useage if you want to make a database for a
    directory which you have no write access to, so you cannot create
    the ID file.)  */
+
 void
 write_id_file (struct idhead *idhp)
 {
@@ -576,6 +620,9 @@ write_id_file (struct idhead *idhp)
   fclose (idhp->idh_FILE);
 }
 
+/* Define primary and secondary hash and comparison functions for the
+   token table.  */
+
 unsigned long
 token_hash_1 (void const *key)
 {
@@ -605,6 +652,9 @@ token_qsort_cmp (void const *x, void const *y)
 
 /****************************************************************************/
 
+/* Advance the tree8 hit signature that corresponds to the file
+   we are now scanning.  */
+
 void
 bump_current_hits_signature (void)
 {
@@ -614,6 +664,8 @@ bump_current_hits_signature (void)
   *hits <<= 1;
 }
 
+/* Initialize the tree8 hit signature for the first file we scan.  */
+
 void
 init_hits_signature (int i)
 {