[PATCH] Handle large requests in nscd

[PATCH] Handle large requests in nscd

[Jakub Jelinek]

Hi!

If nscd is supposed to serve very big requests (e.g. a group with
around 7k members), nscd or the client (or both) can fail to DTRT.
The problem is that nscd in many places doesn't count with
partial reads/writes in read, readv or write.
The following patch introduces __writeall function for the server
and __readall plus __readvall functions that wrap write, __read
and __readv respectively and attempt to read (resp. write) as
many bytes as possible and only stop on failures, EOF condition
or when the whole requested size has been read (resp. written).

2005-02-22  Jakub Jelinek  <jakub@redhat.com>

	* nscd/nscd-client.h: Include sys/uio.h.
	(__readall, __readvall, __writeall): New prototypes.
	* nscd/connections.c (__writeall): New function.
	(handle_request): Use it.
	* nscd/aicache.c (addhstaiX): Likewise.
	* nscd/initgrcache.c (addinitgroupsX): Likewise.
	* nscd/hstcache.c (cache_addhst): Likewise.
	* nscd/grpcache.c (cache_addgr): Likewise.
	* nscd/pwdcache.c (cache_addpw): Likewise.
	* nscd/nscd_helper.c (__readall, __readvall): New functions.
	* nscd/nscd_getai.c (__nscd_getai): Use them.
	* nscd/nscd_getpw_r.c (__nscd_getpw_r): Likewise.
	* nscd/nscd_getgr_r.c (__nscd_getgr_r): Likewise.
	* nscd/nscd_gethst_r.c (__nscd_gethst_r): Likewise.
	* nscd/nscd_initgroups.c (__nscd_getgrouplist): Likewise.

--- libc/nscd/initgrcache.c.jj	2005-01-26 18:36:40.000000000 +0100
+++ libc/nscd/initgrcache.c	2005-02-22 18:39:04.774959626 +0100
@@ -1,5 +1,5 @@
 /* Cache handling for host lookup.
-   Copyright (C) 2004 Free Software Foundation, Inc.
+   Copyright (C) 2004, 2005 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2004.
 
@@ -343,7 +343,7 @@ addinitgroupsX (struct database_dyn *db,
 	     unnecessarily let the receiver wait.  */
 	  assert (fd != -1);
 
-	  written = TEMP_FAILURE_RETRY (write (fd, &dataset->resp, total));
+	  written = __writeall (fd, &dataset->resp, total);
 	}
 
 
--- libc/nscd/hstcache.c.jj	2005-02-22 10:02:31.000000000 +0100
+++ libc/nscd/hstcache.c	2005-02-22 18:30:26.680010234 +0100
@@ -327,7 +327,7 @@ cache_addhst (struct database_dyn *db, i
 	     unnecessarily keep the receiver waiting.  */
 	  assert (fd != -1);
 
-	  written = TEMP_FAILURE_RETRY (write (fd, &dataset->resp, total));
+	  written = __writeall (fd, &dataset->resp, total);
 	}
 
       /* Add the record to the database.  But only if it has not been
--- libc/nscd/grpcache.c.jj	2005-02-22 10:02:30.000000000 +0100
+++ libc/nscd/grpcache.c	2005-02-22 18:29:21.543582609 +0100
@@ -292,7 +292,7 @@ cache_addgr (struct database_dyn *db, in
 	     unnecessarily let the receiver wait.  */
 	  assert (fd != -1);
 
-	  written = TEMP_FAILURE_RETRY (write (fd, &dataset->resp, total));
+	  written = __writeall (fd, &dataset->resp, total);
 	}
 
       /* Add the record to the database.  But only if it has not been
--- libc/nscd/nscd_getgr_r.c.jj	2004-11-10 10:30:32.000000000 +0100
+++ libc/nscd/nscd_getgr_r.c	2005-02-22 18:15:07.008525858 +0100
@@ -1,4 +1,4 @@
-/* Copyright (C) 1998, 1999, 2000, 2002, 2003, 2004
+/* Copyright (C) 1998, 1999, 2000, 2002, 2003, 2004, 2005
    Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Thorsten Kukuk <kukuk@uni-paderborn.de>, 1998.
@@ -202,7 +202,7 @@ nscd_getgr_r (const char *key, size_t ke
 	  total_len = vec[0].iov_len + vec[1].iov_len;
 
 	  /* Get this data.  */
-	  size_t n = TEMP_FAILURE_RETRY (__readv (sock, vec, 2));
+	  size_t n = __readvall (sock, vec, 2);
 	  if (__builtin_expect (n != total_len, 0))
 	    goto out_close;
 	}
@@ -232,8 +232,7 @@ nscd_getgr_r (const char *key, size_t ke
       retval = 0;
       if (gr_name == NULL)
 	{
-	  size_t n = TEMP_FAILURE_RETRY (__read (sock, resultbuf->gr_mem[0],
-						 total_len));
+	  size_t n = __readall (sock, resultbuf->gr_mem[0], total_len);
 	  if (__builtin_expect (n != total_len, 0))
 	    {
 	      /* The `errno' to some value != ERANGE.  */
--- libc/nscd/aicache.c.jj	2005-01-26 18:36:39.000000000 +0100
+++ libc/nscd/aicache.c	2005-02-22 18:28:55.104279879 +0100
@@ -1,5 +1,5 @@
 /* Cache handling for host lookup.
-   Copyright (C) 2004 Free Software Foundation, Inc.
+   Copyright (C) 2004, 2005 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2004.
 
@@ -365,7 +365,7 @@ addhstaiX (struct database_dyn *db, int 
 		     wait.  */
 		  assert (fd != -1);
 
-		  TEMP_FAILURE_RETRY (write (fd, &dataset->resp, total));
+		  __writeall (fd, &dataset->resp, total);
 		}
 
 	      goto out;
--- libc/nscd/nscd_gethst_r.c.jj	2005-02-22 10:02:36.000000000 +0100
+++ libc/nscd/nscd_gethst_r.c	2005-02-22 18:16:03.525475243 +0100
@@ -299,8 +299,7 @@ nscd_gethst_r (const char *key, size_t k
 	      ++n;
 	    }
 
-	  if ((size_t) TEMP_FAILURE_RETRY (__readv (sock, vec, n))
-	      != total_len)
+	  if ((size_t) __readvall (sock, vec, n) != total_len)
 	    goto out_close;
 	}
       else
@@ -329,9 +328,8 @@ nscd_gethst_r (const char *key, size_t k
       /* And finally read the aliases.  */
       if (addr_list == NULL)
 	{
-	  if ((size_t) TEMP_FAILURE_RETRY (__read (sock,
-						   resultbuf->h_aliases[0],
-						   total_len)) == total_len)
+	  if ((size_t) __readall (sock, resultbuf->h_aliases[0], total_len)
+	      == total_len)
 	    {
 	      retval = 0;
 	      *result = resultbuf;
--- libc/nscd/nscd_helper.c.jj	2005-02-07 10:21:15.000000000 +0100
+++ libc/nscd/nscd_helper.c	2005-02-22 18:36:13.605372284 +0100
@@ -34,6 +34,64 @@
 #include "nscd-client.h"
 
 
+ssize_t
+__readall (int fd, void *buf, size_t len)
+{
+  size_t n = len;
+  ssize_t ret;
+  do
+    {
+      ret = TEMP_FAILURE_RETRY (__read (fd, buf, n));
+      if (ret <= 0)
+	break;
+      buf = (char *) buf + ret;
+      n -= ret;
+    }
+  while (n);
+  return ret < 0 ? ret : len - n;
+}
+
+
+ssize_t
+__readvall (int fd, const struct iovec *iov, int iovcnt)
+{
+  ssize_t ret = TEMP_FAILURE_RETRY (__readv (fd, iov, iovcnt));
+  if (ret <= 0)
+    return ret;
+
+  size_t total = 0;
+  int i;
+  for (i = 0; i < iovcnt; ++i)
+    total += iov[i].iov_len;
+
+  if (ret < total)
+    {
+      struct iovec iov_buf[iovcnt], *iovp;
+      ssize_t r = ret;
+      iovp = memcpy (iov_buf, iov, iovcnt * sizeof (*iov));
+      do
+	{
+	  while (iovp->iov_len <= r)
+	    {
+	      r -= iovp->iov_len;
+	      --iovcnt;
+	      ++iovp;
+	    }
+	  iovp->iov_base = (char *) iovp->iov_base + r;
+	  iovp->iov_len -= r;
+	  r = TEMP_FAILURE_RETRY (__readv (fd, iovp, iovcnt));
+	  if (r <= 0)
+	    break;
+	  ret += r;
+	}
+      while (ret < total);
+      if (r < 0)
+	ret = r;
+    }
+  return ret;
+}
+
+
 static int
 open_socket (void)
 {
--- libc/nscd/nscd_getpw_r.c.jj	2004-11-10 10:30:32.000000000 +0100
+++ libc/nscd/nscd_getpw_r.c	2005-02-22 18:16:55.661203796 +0100
@@ -1,4 +1,4 @@
-/* Copyright (C) 1998, 1999, 2003, 2004 Free Software Foundation, Inc.
+/* Copyright (C) 1998, 1999, 2003, 2004, 2005 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Thorsten Kukuk <kukuk@uni-paderborn.de>, 1998.
 
@@ -172,7 +172,7 @@ nscd_getpw_r (const char *key, size_t ke
       retval = 0;
       if (pw_name == NULL)
 	{
-	  ssize_t nbytes = TEMP_FAILURE_RETRY (__read (sock, buffer, total));
+	  ssize_t nbytes = __readall (sock, buffer, total);
 
 	  if (__builtin_expect (nbytes != total, 0))
 	    {
--- libc/nscd/nscd-client.h.jj	2004-11-10 10:30:32.000000000 +0100
+++ libc/nscd/nscd-client.h	2005-02-22 18:39:34.808623306 +0100
@@ -1,4 +1,5 @@
-/* Copyright (c) 1998, 1999, 2000, 2003, 2004 Free Software Foundation, Inc.
+/* Copyright (c) 1998, 1999, 2000, 2003, 2004, 2005
+   Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Thorsten Kukuk <kukuk@suse.de>, 1998.
 
@@ -28,6 +29,7 @@
 #include <sys/types.h>
 #include <atomic.h>
 #include <nscd-types.h>
+#include <sys/uio.h>
 
 
 /* Version number of the daemon interface */
@@ -309,4 +311,13 @@ extern const struct datahead *__nscd_cac
 						   size_t keylen,
 						   const struct mapped_database *mapped);
 
+/* Wrappers around read, readv and write that only read/write less than LEN
+   bytes on error or EOF.  */
+extern ssize_t __readall (int fd, void *buf, size_t len)
+  attribute_hidden;
+extern ssize_t __readvall (int fd, const struct iovec *iov, int iovcnt)
+  attribute_hidden;
+extern ssize_t __writeall (int fd, const void *buf, size_t len)
+  attribute_hidden;
+
 #endif /* nscd.h */
--- libc/nscd/pwdcache.c.jj	2005-02-22 10:02:40.000000000 +0100
+++ libc/nscd/pwdcache.c	2005-02-22 18:27:40.291571183 +0100
@@ -287,7 +287,7 @@ cache_addpw (struct database_dyn *db, in
 	     unnecessarily let the receiver wait.  */
 	  assert (fd != -1);
 
-	  written = TEMP_FAILURE_RETRY (write (fd, &dataset->resp, total));
+	  written = __writeall (fd, &dataset->resp, total);
 	}
 
 
--- libc/nscd/nscd_initgroups.c.jj	2005-02-08 17:17:18.000000000 +0100
+++ libc/nscd/nscd_initgroups.c	2005-02-22 18:17:38.709548411 +0100
@@ -110,9 +110,8 @@ __nscd_getgrouplist (const char *user, g
       if (respdata == NULL)
 	{
 	  /* Read the data from the socket.  */
-	  if ((size_t) TEMP_FAILURE_RETRY (__read (sock, *groupsp,
-						   initgr_resp->ngrps
-						   * sizeof (gid_t)))
+	  if ((size_t) __readall (sock, *groupsp, initgr_resp->ngrps
+						  * sizeof (gid_t))
 	      == initgr_resp->ngrps * sizeof (gid_t))
 	    retval = initgr_resp->ngrps;
 	}
--- libc/nscd/nscd_getai.c.jj	2005-02-08 17:17:18.000000000 +0100
+++ libc/nscd/nscd_getai.c	2005-02-22 18:09:44.607856677 +0100
@@ -119,8 +119,7 @@ __nscd_getai (const char *key, struct ns
       if (respdata == NULL)
 	{
 	  /* Read the data from the socket.  */
-	  if ((size_t) TEMP_FAILURE_RETRY (__read (sock, resultbuf + 1,
-						   datalen)) == datalen)
+	  if ((size_t) __readall (sock, resultbuf + 1, datalen) == datalen)
 	    {
 	      retval = 0;
 	      *result = resultbuf;
--- libc/nscd/connections.c.jj	2005-02-07 10:21:15.000000000 +0100
+++ libc/nscd/connections.c	2005-02-22 18:34:29.444878715 +0100
@@ -181,6 +181,24 @@ static int sock;
 unsigned long int client_queued;
 
 
+ssize_t
+__writeall (int fd, const void *buf, size_t len)
+{
+  size_t n = len;
+  ssize_t ret;
+  do
+    {
+      ret = TEMP_FAILURE_RETRY (write (fd, buf, n));
+      if (ret <= 0)
+	break;
+      buf = (const char *) buf + ret;
+      n -= ret;
+    }
+  while (n);
+  return ret < 0 ? ret : len - n;
+}
+
+
 /* Initialize database information structures.  */
 void
 nscd_init (void)
@@ -691,7 +709,7 @@ cannot handle old request version %d; cu
       if (cached != NULL)
 	{
 	  /* Hurray it's in the cache.  */
-	  if (TEMP_FAILURE_RETRY (write (fd, cached->data, cached->recsize))
+	  if (__writeall (fd, cached->data, cached->recsize)
 	      != cached->recsize
 	      && __builtin_expect (debug_level, 0) > 0)
 	    {

	Jakub

Re: [PATCH] Handle large requests in nscd

[Ulrich Drepper]

[-- Attachment #1: Type: text/plain, Size: 179 bytes --]

Applied.  Although I changed __writeall to writeall.  It's all in nscd
itself.

--
➧ Ulrich Drepper ➧ Red Hat, Inc. ➧ 444 Castro St ➧ Mountain View, CA ❖

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 252 bytes --]