Subject: Patch to inn-1.5.1's nnrpd to mmap() the active file shared+ro

From: [email-masked] (Matt Dillon)
Date: 4 Feb 1997 00:55:02 -0800

This patch uses mmap() to map the active file shared + read-only.
    For newsreader machines with large (~ 1MByte) active files, this
    patch more then halves the per-nnrpd process memory usage.  It is
    roughly equivalent to the shared-active mods that have been
    floating around, but approaches the problem somewhat differently.

    I don't know which is better, or whether it even matters.  I do know
    that this patch allows our newsreader machine to support nearly twice
    as many nnrpd's as it would otherwise be able to... around 
    500 nnrpd processes on a 256 MByte pentium-pro 200 running FreeBSD.

    The patch has been reasonably well tested, and since it restricts
    itself to a read-only map, it ought to work on any machine that
    supports mmap().

					-Matt

diff -r -c nnrpd/commands.c nnrpd-matt/commands.c
*** nnrpd/commands.c	Tue Dec 17 06:40:40 1996
--- nnrpd-matt/commands.c	Mon Feb  3 21:59:33 1997
***************
*** 322,329 ****
  	    if (gp) {
  		Reply("%d list:\r\n", NNTP_LIST_FOLLOWS_VAL);
  		Printf("%s %ld %ld %c%s\r\n.\r\n",
! 		      gp->Name, (long)gp->High, (long)gp->Low,
! 		      gp->Flag, gp->Alias ? gp->Alias : "");
  		return;
  	    }
  	    wildarg = av[2];
--- 322,329 ----
  	    if (gp) {
  		Reply("%d list:\r\n", NNTP_LIST_FOLLOWS_VAL);
  		Printf("%s %ld %ld %c%s\r\n.\r\n",
! 		      GPNAME(gp), (long)GPHIGH(gp), (long)GPLOW(gp),
! 		      GPFLAG(gp), GPALIAS(gp) ? GPALIAS(gp) : "");
  		return;
  	    }
  	    wildarg = av[2];
***************
*** 528,535 ****
  		continue;
  	}
  	Printf("%s %ld %ld %c%s\r\n",
! 	    p, (long)gp->High, (long)gp->Low,
! 	    gp->Flag, gp->Alias ? gp->Alias : "");
      }
      QIOclose(qp);
      Printf(".\r\n");
--- 528,535 ----
  		continue;
  	}
  	Printf("%s %ld %ld %c%s\r\n",
! 	    p, (long)GPHIGH(gp), (long)GPLOW(gp),
! 	    GPFLAG(gp), GPALIAS(gp) ? GPALIAS(gp) : "");
      }
      QIOclose(qp);
      Printf(".\r\n");
diff -r -c nnrpd/group.c nnrpd-matt/group.c
*** nnrpd/group.c	Tue Dec 17 06:40:40 1996
--- nnrpd-matt/group.c	Mon Feb  3 22:32:25 1997
***************
*** 8,13 ****
--- 8,14 ----
  #include "clibrary.h"
  #include "nnrpd.h"
  #include "mydir.h"
+ #include <sys/mman.h>

  /*
***************
*** 20,28 ****
--- 21,32 ----
  #define GRP_BUCKET(j)	&GRPtable[j & (GRP_SIZE - 1)]
  
  typedef struct _GRPHASH {
+ #ifdef NOTDEF
      int		Size;
      int		Used;
      GROUPENTRY	**Groups;
+ #endif
+     GROUPENTRY	*First;
  } GRPHASH;

***************
*** 42,57 ****
      register char		*p;
      register unsigned int	j;
      register int		i;
!     register GROUPENTRY		**gpp;
      GRPHASH			*htp;
      char			c;
  
      /* SUPPRESS 6 *//* Over/underflow from plus expression */
      GRP_HASH(group, p, j);
      htp = GRP_BUCKET(j);
!     for (c = *group, gpp = htp->Groups, i = htp->Used; --i >= 0; gpp++)
! 	if (c == gpp[0]->Name[0] && EQ(group, gpp[0]->Name))
! 	    return gpp[0];
      return NULL;
  }
  
--- 46,62 ----
      register char		*p;
      register unsigned int	j;
      register int		i;
!     register GROUPENTRY		*gp;
      GRPHASH			*htp;
      char			c;
  
      /* SUPPRESS 6 *//* Over/underflow from plus expression */
      GRP_HASH(group, p, j);
      htp = GRP_BUCKET(j);
!     for (c = *group, gp = htp->First; gp; gp = gp->Next) {
!         if (c == GPNAME(gp)[0] && EQ(group, GPNAME(gp)))
!             return gp;
!     }
      return NULL;
  }
  
***************
*** 69,74 ****
--- 74,83 ----
      GRPbuckets = GRPsize / GRP_SIZE;
      if (GRPbuckets == 0)
  	GRPbuckets = 1;
+     for (i = 0; i < GRP_SIZE; ++i)
+ 	GRPtable[i].First = NULL;
+ 
+ #ifdef NOTDEF
      if (GRPtable[0].Groups)
  	for (i = GRP_SIZE, htp = GRPtable; --i >= 0; htp++)
  	    htp->Used = 0;
***************
*** 78,99 ****
  	    htp->Groups = NEW(GROUPENTRY*, htp->Size);
  	    htp->Used = 0;
  	}
  
      /* Now put all groups into the hash table. */
      for (i = GRPsize, gp = GRPentries; --i >= 0; gp++) {
  	/* SUPPRESS 6 *//* Over/underflow from plus expression */
! 	GRP_HASH(gp->Name, p, j);
  	htp = GRP_BUCKET(j);
  	if (htp->Used >= htp->Size) {
  	    htp->Size += GRPbuckets;
  	    RENEW(htp->Groups, GROUPENTRY*, htp->Size);
  	}
  	htp->Groups[htp->Used++] = gp;
      }
  
      /* Note that we don't sort the buckets. */
  }

  /*
  **  Read the active file into memory, sort it, and set the number of
--- 87,232 ----
  	    htp->Groups = NEW(GROUPENTRY*, htp->Size);
  	    htp->Used = 0;
  	}
+ #endif
  
      /* Now put all groups into the hash table. */
      for (i = GRPsize, gp = GRPentries; --i >= 0; gp++) {
  	/* SUPPRESS 6 *//* Over/underflow from plus expression */
! 	GRP_HASH(GPNAME(gp), p, j);
  	htp = GRP_BUCKET(j);
+ #ifdef NOTDEF
  	if (htp->Used >= htp->Size) {
  	    htp->Size += GRPbuckets;
  	    RENEW(htp->Groups, GROUPENTRY*, htp->Size);
  	}
  	htp->Groups[htp->Used++] = gp;
+ #endif
+ 	gp->Next = htp->First;
+ 	htp->First = gp;
+ 	
      }
  
      /* Note that we don't sort the buckets. */
  }
  
+ long XRSize = -1;
+ 
+ char *
+ XReadInFile(const char *fileName, struct stat *pst)
+ {
+     int fd;
+     char *ptr = NULL;
+     struct stat st;
+ 
+     if (pst == NULL)
+         pst = &st;
+ 
+     fd = open(fileName, O_RDONLY);
+     if (fd >= 0) {
+         if (fstat(fd, pst) == 0) {
+             ptr = (char *)mmap((caddr_t)NULL, pst->st_size + 1, PROT_READ, MAP_SHARED, fd, 0);
+             if (ptr != (char *)-1) {
+                 XRSize = pst->st_size + 1;
+                 close(fd);
+                 /* syslog(L_ERROR, "%s mapped %d", ClientHost, pst->st_size + 1);*/
+             } else {
+                 close(fd);
+                 return(ReadInFile(fileName, pst));
+             }
+         } else {
+             close(fd);
+         }
+     }
+     return(ptr);
+ }
+ 
+ void
+ XDISPOSE(char *active)
+ {
+     if (XRSize >= 0) {
+         munmap(active, XRSize);
+         XRSize = -1;
+     } else {
+         DISPOSE(active);
+     }
+ }
+ 
+ char *
+ GPNAME(GROUPENTRY *gp)
+ {
+     const char *p = strchr(gp->Ptr, ' ');
+     static char GPBuf[1024];
+ 
+     GPBuf[0] = 0;
+     if (p && p - gp->Ptr > 0 && p - gp->Ptr < sizeof(GPBuf) - 1) {
+         bcopy(gp->Ptr, GPBuf, p - gp->Ptr);
+         GPBuf[p - gp->Ptr] = 0;
+     }
+     return(GPBuf);
+ }
+ 
+ ARTNUM
+ GPHIGH(GROUPENTRY *gp)
+ {
+     const char *p = strchr(gp->Ptr, ' ');
+     long n = 0;
+     if (p)
+         n = strtol(p + 1, NULL, 10);
+     return(n);
+ }
+ 
+ ARTNUM
+ GPLOW(GROUPENTRY *gp)
+ {
+     const char *p = strchr(gp->Ptr, ' ');
+     long n = 0;
+     if (p) {
+         if ((p = strchr(p + 1, ' ')) != NULL)
+             n = strtol(p + 1, NULL, 10);
+     }
+     return(n);
+ }
+ 
+ 
+ char
+ GPFLAG(GROUPENTRY *gp)
+ {
+     const char *p = strchr(gp->Ptr, ' ');
+     char c = 'n';
+ 
+     if (p) {
+         if ((p = strchr(p + 1, ' ')) != NULL) {
+             if ((p = strchr(p + 1, ' ')) != NULL) {
+                 c = p[1];
+             }
+         }
+     }
+     return(c);
+ }
+ 
+ 
+ char *
+ GPALIAS(GROUPENTRY *gp)
+ {
+     const char *p = strchr(gp->Ptr, ' ');
+     static char GPBuf[1024];
+ 
+     GPBuf[0] = 0;
+ 
+     if (p) {
+         if ((p = strchr(p + 1, ' ')) != NULL) {
+             if ((p = strchr(p + 1, ' ')) != NULL) {
+                 if (p[0] == NF_FLAG_ALIAS &&
+                     p[1] != '\n' &&
+                     strlen(p + 1) < sizeof(GPBuf) - 1
+                 ) {
+                     strcpy(GPBuf, p + 1);
+                 }
+             }
+         }
+     }
+     return(GPBuf);
+ }
  
  /*
  **  Read the active file into memory, sort it, and set the number of
***************
*** 110,121 ****
  
      /* If re-scanning, free previous groups. */
      if (active != NULL) {
! 	DISPOSE(active);
  	DISPOSE(GRPentries);
      }
  
      /* Get the new file. */
!     active = ReadInFile(ACTIVE, (struct stat *)NULL);
      if (active == NULL) {
  	syslog(L_ERROR, "%s cant read %s %m", ClientHost, ACTIVE);
  	return FALSE;
--- 243,254 ----
  
      /* If re-scanning, free previous groups. */
      if (active != NULL) {
! 	XDISPOSE(active);
  	DISPOSE(GRPentries);
      }
  
      /* Get the new file. */
!     active = XReadInFile(ACTIVE, (struct stat *)NULL);
      if (active == NULL) {
  	syslog(L_ERROR, "%s cant read %s %m", ClientHost, ACTIVE);
  	return FALSE;
***************
*** 128,168 ****
      /* Fill in the group array. */
      GRPentries = NEW(GROUPENTRY, i);
      for (i = 0, gp = GRPentries, p = active; *p; i++, gp++, p = q + 1) {
  	gp->Name = p;
  	if ((p = strchr(p, ' ')) == NULL) {
  	    syslog(L_ERROR, "%s internal no_space1 \"%.20s...\"",
! 		ClientHost, gp->Name);
  	    return FALSE;
  	}
  	*p++ = '\0';
  
  	/* Get the high mark. */
  	if ((q = strchr(p, ' ')) == NULL) {
  	    syslog(L_ERROR, "%s internal no_space2 \"%.20s...\"",
! 		ClientHost, gp->Name);
  	    return FALSE;
  	}
  	*q++ = '\0';
  	gp->High = atol(p);
  
  	/* Get the low mark. */
  	if ((p = strchr(q, ' ')) == NULL) {
  	    syslog(L_ERROR, "%s internal no_space3 \"%.20s...\"",
! 		ClientHost, gp->Name);
  	    return FALSE;
  	}
  	*p++ = '\0';
  	gp->Low = atol(q);
  
  	/* Kill the newline. */
  	if ((q = strchr(p, '\n')) == NULL) {
  	    syslog(L_ERROR, "%s internal newline \"%.20s...\"",
! 		ClientHost, gp->Name);
  	    return FALSE;
  	}
  	*q = '\0';
  	gp->Flag = *p;
  	gp->Alias = gp->Flag == NF_FLAG_ALIAS ? p + 1 : NULL;
      }
  
      GRPsize = i;
--- 261,313 ----
      /* Fill in the group array. */
      GRPentries = NEW(GROUPENTRY, i);
      for (i = 0, gp = GRPentries, p = active; *p; i++, gp++, p = q + 1) {
+ 	gp->Ptr = p;
+ 
+ #ifdef NOTDEF
  	gp->Name = p;
+ #endif
  	if ((p = strchr(p, ' ')) == NULL) {
  	    syslog(L_ERROR, "%s internal no_space1 \"%.20s...\"",
! 		ClientHost, GPNAME(gp));
  	    return FALSE;
  	}
+ #ifdef NOTDEF
  	*p++ = '\0';
+ #endif
  
  	/* Get the high mark. */
  	if ((q = strchr(p, ' ')) == NULL) {
  	    syslog(L_ERROR, "%s internal no_space2 \"%.20s...\"",
! 		ClientHost, GPNAME(gp));
  	    return FALSE;
  	}
+ #ifdef NOTDEF
  	*q++ = '\0';
  	gp->High = atol(p);
+ #endif
  
  	/* Get the low mark. */
  	if ((p = strchr(q, ' ')) == NULL) {
  	    syslog(L_ERROR, "%s internal no_space3 \"%.20s...\"",
! 		ClientHost, GPNAME(gp));
  	    return FALSE;
  	}
+ #ifdef NOTDEF
  	*p++ = '\0';
  	gp->Low = atol(q);
+ #endif
  
  	/* Kill the newline. */
  	if ((q = strchr(p, '\n')) == NULL) {
  	    syslog(L_ERROR, "%s internal newline \"%.20s...\"",
! 		ClientHost, GPNAME(gp));
  	    return FALSE;
  	}
+ #ifdef NOTDEF
  	*q = '\0';
  	gp->Flag = *p;
  	gp->Alias = gp->Flag == NF_FLAG_ALIAS ? p + 1 : NULL;
+ #endif
      }
  
      GRPsize = i;
diff -r -c nnrpd/nnrpd.h nnrpd-matt/nnrpd.h
*** nnrpd/nnrpd.h	Tue Dec 17 06:40:40 1996
--- nnrpd-matt/nnrpd.h	Mon Feb  3 22:07:38 1997
***************
*** 49,59 ****
--- 49,63 ----
  **  A group entry.
  */
  typedef struct _GROUPENTRY {
+ #ifdef NOTDEF
      char	*Name;
      ARTNUM	High;
      ARTNUM	Low;
      char	Flag;
      char	*Alias;
+ #endif
+     char        *Ptr;   
+     struct _GROUPENTRY *Next;
  } GROUPENTRY;

***************
*** 142,144 ****
--- 146,155 ----
  #if	defined(VAR_VARARGS)
  extern void		Reply();
  #endif	/* defined(VAR_VARARGS) */
+ 
+ char *GPNAME(GROUPENTRY *gp);   
+ ARTNUM GPLOW(GROUPENTRY *gp);
+ ARTNUM GPHIGH(GROUPENTRY *gp);
+ char GPFLAG(GROUPENTRY *gp);
+ char *GPALIAS(GROUPENTRY *gp); 
+ 
diff -r -c nnrpd/post.c nnrpd-matt/post.c
*** nnrpd/post.c	Tue Dec 17 06:40:40 1996
--- nnrpd-matt/post.c	Mon Feb  3 22:00:04 1997
***************
*** 589,608 ****
  	    continue;
  	FoundOne = TRUE;
  	DDcheck(h, p);
! 	switch (gp->Flag) {
  	case NF_FLAG_OK:
  	    break;
  	case NF_FLAG_MODERATED:
  	    if (!approved) {
  		DISPOSE(groups);
  		DISPOSE(DDend(h));
! 		return MailArticle(gp->Name, article);
  	    }
  	    break;
  	case NF_FLAG_IGNORE:
  	case NF_FLAG_NOLOCAL:
  	    (void)sprintf(Error, "Postings to \"%s\" are not allowed here.",
! 		    gp->Name);
  	    break;
  	case NF_FLAG_EXCLUDED:
  	    /* Do NOT return an error. */
--- 589,608 ----
  	    continue;
  	FoundOne = TRUE;
  	DDcheck(h, p);
! 	switch (GPFLAG(gp)) {
  	case NF_FLAG_OK:
  	    break;
  	case NF_FLAG_MODERATED:
  	    if (!approved) {
  		DISPOSE(groups);
  		DISPOSE(DDend(h));
! 		return MailArticle(GPNAME(gp), article);
  	    }
  	    break;
  	case NF_FLAG_IGNORE:
  	case NF_FLAG_NOLOCAL:
  	    (void)sprintf(Error, "Postings to \"%s\" are not allowed here.",
! 		    GPNAME(gp));
  	    break;
  	case NF_FLAG_EXCLUDED:
  	    /* Do NOT return an error. */
***************
*** 610,616 ****
  	case NF_FLAG_ALIAS:
  	    (void)sprintf(Error,
  		    "The newsgroup \"%s\" has been renamed to \"%s\".\n",
! 		    p, gp->Alias);
  	    break;
  	}
          if (PERMspecified) {
--- 610,616 ----
  	case NF_FLAG_ALIAS:
  	    (void)sprintf(Error,
  		    "The newsgroup \"%s\" has been renamed to \"%s\".\n",
! 		    p, GPALIAS(gp));
  	    break;
  	}
          if (PERMspecified) {