[PATCH 0/3] Signal-based file switching support

[PATCH 0/3] Signal-based file switching support

[Masami Hiramatsu]

Hi,

This patch series adds signal-based file switching support on systemtap
runtime. This feature allows administators to move their stap logs to
other storage safely on file-flight recorder mode (-F and -o option
specified).

E.g.

$ stap -F -o logfile -S 100 systemlog.stp
12345
$ ls
logfile.0
<<something happened, or backup routine work>>
$ kill -USR1 12345
$ ls
logfile.0 logfile.1
$ mv logfile.0 /mnt/backup/ 

Thank you,

---
 
Masami Hiramatsu (3):
      Add signal based file switching testcase
      Signal-based file switching support for old relay
      Signal-based file switching support for relay/ring buffer.


 runtime/staprun/relay.c                 |   60 ++++++++++++++++++----
 runtime/staprun/relay_old.c             |   83 +++++++++++++++++++++++++++----
 testsuite/systemtap.base/flightrec1.exp |   12 ++++
 testsuite/systemtap.base/flightrec4.exp |   56 +++++++++++++++++++++
 testsuite/systemtap.base/flightrec5.exp |   64 ++++++++++++++++++++++++
 5 files changed, 251 insertions(+), 24 deletions(-)
 create mode 100644 testsuite/systemtap.base/flightrec4.exp
 create mode 100644 testsuite/systemtap.base/flightrec5.exp

-- 
Masami Hiramatsu
 
Software Engineer
Hitachi Computer Products (America), Inc.
Software Solutions Division
 
e-mail: mhiramat@redhat.com

[PATCH 2/3] Signal-based file switching support for old relay

[Masami Hiramatsu]

* runtime/staprun/relay_old.c (switch_oldoutfile): New function for file
  switching.
  (process_subbufs): Use switch_oldoutfile.
  (reader_thread): Block SIGUSR1 and SIGUSR2 in default, and use ppoll()
  instead of poll() for receiving SIGUSR2.
  (switchfile_handler): Send SIGUSR2 signal to reader threads for file
  switching.
  (init_oldrelayfs): Assign switchfile_handler to SIGUSR1.
---

 runtime/staprun/relay_old.c |   83 +++++++++++++++++++++++++++++++++++++------
 1 files changed, 71 insertions(+), 12 deletions(-)

diff --git a/runtime/staprun/relay_old.c b/runtime/staprun/relay_old.c
index 0254173..64bdf1c 100644
--- a/runtime/staprun/relay_old.c
+++ b/runtime/staprun/relay_old.c
@@ -19,6 +19,7 @@ static int proc_fd[NR_CPUS];
 static FILE *percpu_tmpfile[NR_CPUS];
 static char *relay_buffer[NR_CPUS];
 static pthread_t reader[NR_CPUS];
+static int switch_file[NR_CPUS];
 static int bulkmode = 0;
 unsigned subbuf_size = 0;
 unsigned n_subbufs = 0;
@@ -214,6 +215,22 @@ err1:
 
 }
 
+static int switch_oldoutfile(int cpu, struct switchfile_ctrl_block *scb)
+{
+	dbug(3, "thread %d switching file\n", cpu);
+	if (percpu_tmpfile[cpu])
+		fclose(percpu_tmpfile[cpu]);
+	else
+		close(out_fd[cpu]);
+	scb->fnum ++;
+	if (fnum_max && scb->fnum == fnum_max)
+		scb->rmfile = 1;
+	if (open_oldoutfile(scb->fnum, cpu, scb->rmfile) < 0) {
+		perr("Couldn't open file for cpu %d, exiting.", cpu);
+		return -1;
+	}
+	return 0;
+}
 /**
  *	process_subbufs - write ready subbufs to disk
  */
@@ -238,11 +255,7 @@ static int process_subbufs(struct _stp_buf_info *info,
 		len = (subbuf_size - sizeof(padding)) - padding;
 		scb->wsize += len;
 		if (fsize_max && scb->wsize > fsize_max) {
-			fclose(percpu_tmpfile[cpu]);
-			scb->fnum ++;
-			if (fnum_max && scb->fnum == fnum_max)
-				scb->rmfile = 1;
-			if (open_oldoutfile(scb->fnum, cpu, scb->rmfile) < 0) {
+			if (switch_oldoutfile(cpu, scb) < 0) {
 				perr("Couldn't open file for cpu %d, exiting.", cpu);
 				return -1;
 			}
@@ -264,6 +277,8 @@ static int process_subbufs(struct _stp_buf_info *info,
 /**
  *	reader_thread - per-cpu channel buffer reader
  */
+static void empty_handler(int __attribute__((unused)) sig) { /* do nothing */  }
+
 static void *reader_thread(void *data)
 {
 	int rc;
@@ -272,7 +287,23 @@ static void *reader_thread(void *data)
 	struct _stp_consumed_info consumed_info;
 	unsigned subbufs_consumed;
 	cpu_set_t cpu_mask;
+	struct timespec tim = {.tv_sec=0, .tv_nsec=200000000}, *timeout = &tim;
 	struct switchfile_ctrl_block scb = {0, 0, 0};
+	sigset_t sigs;
+	struct sigaction sa;
+
+	sigemptyset(&sigs);
+	sigaddset(&sigs,SIGUSR1);
+	sigaddset(&sigs,SIGUSR2);
+	pthread_sigmask(SIG_BLOCK, &sigs, NULL);
+
+	sigfillset(&sigs);
+	sigdelset(&sigs,SIGUSR2);
+	
+	sa.sa_handler = empty_handler;
+        sa.sa_flags = 0;
+	sigemptyset(&sa.sa_mask);
+        sigaction(SIGUSR2, &sa, NULL);
 
 	CPU_ZERO(&cpu_mask);
 	CPU_SET(cpu, &cpu_mask);
@@ -281,9 +312,17 @@ static void *reader_thread(void *data)
 
 	pollfd.fd = relay_fd[cpu];
 	pollfd.events = POLLIN;
+#ifdef NEED_PPOLL
+	/* Without a real ppoll, there is a small race condition that could */
+	/* block ppoll(). So use a timeout to prevent that. */
+	timeout->tv_sec = 10;
+	timeout->tv_nsec = 0;
+#else
+	timeout = NULL;
+#endif
 
 	do {
-		rc = poll(&pollfd, 1, -1);
+                rc = ppoll(&pollfd, 1, timeout, &sigs);
 		if (rc < 0) {
 			if (errno != EINTR) {
 				_perr("poll error");
@@ -292,6 +331,12 @@ static void *reader_thread(void *data)
 			err("WARNING: poll warning: %s\n", strerror(errno));
 			rc = 0;
 		}
+		if (switch_file[cpu]) {
+			switch_file[cpu] = 0;
+			if (switch_oldoutfile(cpu, &scb) < 0)
+				break;
+			scb.wsize = 0;
+		}
 
 		rc = read(proc_fd[cpu], &status[cpu].info, sizeof(struct _stp_buf_info));
 		rc = process_subbufs(&status[cpu].info, &scb);
@@ -324,12 +369,7 @@ int write_realtime_data(void *data, ssize_t nb)
 	ssize_t bw;
 	global_scb.wsize += nb;
 	if (fsize_max && global_scb.wsize > fsize_max) {
-		close(out_fd[0]);
-		global_scb.fnum++;
-		if (fnum_max && global_scb.fnum == fnum_max)
-			global_scb.rmfile = 1;
-		if (open_oldoutfile(global_scb.fnum, 0,
-				    global_scb.rmfile) < 0) {
+		if (switch_oldoutfile(0, &global_scb) < 0) {
 			perr("Couldn't open file, exiting.");
 			return -1;
 		}
@@ -343,6 +383,19 @@ int write_realtime_data(void *data, ssize_t nb)
 	return bw != nb;
 }
 
+static void switchfile_handler(int sig)
+{
+	int i;
+	dbug(3, "file switching signal %d received\n", sig);
+	for (i = 0; i < ncpus; i++) {
+		if (reader[i]) {
+			switch_file[i] = 1;
+			pthread_kill(reader[i], SIGUSR2);
+		} else
+			break;
+	}
+}
+
 /**
  *	init_relayfs - create files and threads for relayfs processing
  *
@@ -353,6 +406,12 @@ int init_oldrelayfs(void)
 	int i, j;
 	struct statfs st;
 	char relay_filebase[PATH_MAX], proc_filebase[PATH_MAX];
+	struct sigaction sa;
+
+	sa.sa_handler = switchfile_handler;
+	sa.sa_flags = 0;
+	sigemptyset(&sa.sa_mask);
+	sigaction(SIGUSR1, &sa, NULL);
 
 	dbug(2, "initializing relayfs.n_subbufs=%d subbuf_size=%d\n", n_subbufs, subbuf_size);
 


-- 
Masami Hiramatsu

Software Engineer
Hitachi Computer Products (America), Inc.
Software Solutions Division

e-mail: mhiramat@redhat.com

[PATCH 3/3] Add signal based file switching testcase

[Masami Hiramatsu]

* testsuite/systemtap.base/flightrec1.exp: Add signal file switching testcase.
* testsuite/systemtap.base/flightrec4.exp: New test for signal file switching
  with file number limits.
* testsuite/systemtap.base/flightrec5.exp: New test for signal file switching
  with file number limits on bulk mode.
---

 testsuite/systemtap.base/flightrec1.exp |   12 +++++-
 testsuite/systemtap.base/flightrec4.exp |   56 +++++++++++++++++++++++++++
 testsuite/systemtap.base/flightrec5.exp |   64 +++++++++++++++++++++++++++++++
 3 files changed, 131 insertions(+), 1 deletions(-)
 create mode 100644 testsuite/systemtap.base/flightrec4.exp
 create mode 100644 testsuite/systemtap.base/flightrec5.exp

diff --git a/testsuite/systemtap.base/flightrec1.exp b/testsuite/systemtap.base/flightrec1.exp
index c32a77f..eb7f26c 100644
--- a/testsuite/systemtap.base/flightrec1.exp
+++ b/testsuite/systemtap.base/flightrec1.exp
@@ -30,7 +30,8 @@ expect {
 }
 wait
 
-exec kill -TERM $pid
+# switch file
+exec kill -USR1 $pid
 
 # check output file
 if {[catch {exec rm $test.out}]} {
@@ -40,4 +41,13 @@ if {[catch {exec rm $test.out}]} {
   pass "$test (output file)"
 }
 
+exec kill -TERM $pid
+
+# check switched output file
+if {[catch {exec rm $test.out.1}]} {
+  fail "$test (failed to switch output file)"
+  return -1
+} else {
+  pass "$test (switch output file)"
+}
 
diff --git a/testsuite/systemtap.base/flightrec4.exp b/testsuite/systemtap.base/flightrec4.exp
new file mode 100644
index 0000000..36c02d5
--- /dev/null
+++ b/testsuite/systemtap.base/flightrec4.exp
@@ -0,0 +1,56 @@
+set test "flightrec4"
+if {![installtest_p]} { untested $test; return }
+
+# run stapio in background mode with number limit
+spawn stap -F -S 1,2 -o $test.out -we {probe begin {}}
+# check whether stap outputs stapio pid
+set pid 0
+expect {
+  -timeout 240
+  -re {([0-9]+)\r\n} {
+      pass "$test (flight recorder option)"
+      set pid $expect_out(1,string)
+      exp_continue}
+  timeout { fail "$test (timeout)" }
+  eof { }
+}
+wait
+if {$pid == 0} {
+  fail "$test (no pid)"
+  return -1
+}
+
+# switch file to .1
+exec kill -USR1 $pid
+
+# check output file
+if {[catch {exec rm $test.out.0}]} {
+  fail "$test (no output file)"
+  return -1
+} else {
+  pass "$test (output file)"
+}
+
+# switch file to .2
+exec kill -USR1 $pid
+# switch file to .3 (this time, .1 file should be deleted)
+exec kill -USR1 $pid
+
+# check switched output file
+if {[catch {exec rm $test.out.1}]} {
+  pass "$test (old output file is removed)"
+} else {
+  fail "$test (failed to remove output file)"
+  return -1
+}
+
+exec kill -TERM $pid
+
+# check switched output file
+if {[catch {exec rm $test.out.2 $test.out.3}]} {
+  fail "$test (failed to switch output file)"
+  return -1
+} else {
+  pass "$test (switch output file)"
+}
+
diff --git a/testsuite/systemtap.base/flightrec5.exp b/testsuite/systemtap.base/flightrec5.exp
new file mode 100644
index 0000000..e24e6e5
--- /dev/null
+++ b/testsuite/systemtap.base/flightrec5.exp
@@ -0,0 +1,64 @@
+set test "flightrec5"
+if {![installtest_p]} { untested $test; return }
+
+# run stapio in background mode with number limit and bulk mode
+spawn stap -F -S 1,2 -b -o $test.out -we {probe begin {}}
+# check whether stap outputs stapio pid
+set pid 0
+expect {
+  -timeout 240
+  -re {([0-9]+)\r\n} {
+      pass "$test (flight recorder option)"
+      set pid $expect_out(1,string)
+      exp_continue}
+  timeout { fail "$test (timeout)" }
+  eof { }
+}
+wait
+if {$pid == 0} {
+  fail "$test (no pid)"
+  return -1
+}
+
+# switch file to .1
+exec kill -USR1 $pid
+
+# check output file
+eval set outfile {[glob -nocomplain $test.out_cpu*.0]}
+if {$outfile == ""} {
+  fail "$test (no output file) $outfile"
+  exec kill -TERM $pid
+  return -1
+} else {
+  pass "$test (output file)"
+}
+eval exec rm $outfile
+
+print "pid = $pid"
+# switch file to .2
+exec kill -USR1 $pid
+# switch file to .3 (this time, .1 file should be deleted)
+exec kill -USR1 $pid
+
+exec kill -TERM $pid
+
+# check switched output file
+eval set outfile {[glob -nocomplain $test.out_cpu*.1]}
+if {$outfile == ""} {
+  pass "$test (old output file is removed)"
+} else {
+  fail "$test (failed to remove output file)"
+  eval exec rm $outfile
+  return -1
+}
+
+# check switched output file
+eval set outfile {[glob -nocomplain $test.out_cpu*.*]}
+if {$outfile == ""} {
+  fail "$test (failed to switch output file)"
+  return -1
+} else {
+  pass "$test (switch output file)"
+}
+eval exec rm $outfile
+


-- 
Masami Hiramatsu

Software Engineer
Hitachi Computer Products (America), Inc.
Software Solutions Division

e-mail: mhiramat@redhat.com

[PATCH 1/3] Signal-based file switching support for relay/ring buffer.

[Masami Hiramatsu]

* runtime/staprun/relay_old.c (switch_outfile): New function for file
  switching.
  (reader_thread): Use switch_oldoutfile and block SIGUSR1 and SIGUSR2
  in default.
  (switchfile_handler): Send SIGUSR2 signal to reader threads for file
  switching.
  (init_relayfs): Assign switchfile_handler to SIGUSR1.
---

 runtime/staprun/relay.c |   60 ++++++++++++++++++++++++++++++++++++++---------
 1 files changed, 49 insertions(+), 11 deletions(-)

diff --git a/runtime/staprun/relay.c b/runtime/staprun/relay.c
index f4aa139..362a251 100644
--- a/runtime/staprun/relay.c
+++ b/runtime/staprun/relay.c
@@ -15,6 +15,7 @@
 int out_fd[NR_CPUS];
 static pthread_t reader[NR_CPUS];
 static int relay_fd[NR_CPUS];
+static int switch_file[NR_CPUS];
 static int bulkmode = 0;
 static volatile int stop_threads = 0;
 static time_t *time_backlog[NR_CPUS];
@@ -107,6 +108,21 @@ static int open_outfile(int fnum, int cpu, int remove_file)
 	return 0;
 }
 
+static int switch_outfile(int cpu, int *fnum)
+{
+	int remove_file = 0;
+
+	dbug(3, "thread %d switching file\n", cpu);
+	close(out_fd[cpu]);
+	*fnum += 1;
+	if (fnum_max && *fnum >= fnum_max)
+		remove_file = 1;
+	if (open_outfile(*fnum, cpu, remove_file) < 0) {
+		perr("Couldn't open file for cpu %d, exiting.", cpu);
+		return -1;
+	}
+	return 0;
+}
 /**
  *	reader_thread - per-cpu channel buffer reader
  */
@@ -122,9 +138,9 @@ static void *reader_thread(void *data)
 	struct sigaction sa;
 	off_t wsize = 0;
 	int fnum = 0;
-	int remove_file = 0;
 
 	sigemptyset(&sigs);
+	sigaddset(&sigs,SIGUSR1);
 	sigaddset(&sigs,SIGUSR2);
 	pthread_sigmask(SIG_BLOCK, &sigs, NULL);
 
@@ -156,6 +172,7 @@ static void *reader_thread(void *data)
 	pollfd.events = POLLIN;
 
         do {
+		dbug(3, "thread %d start ppoll\n", cpu);
                 rc = ppoll(&pollfd, 1, timeout, &sigs);
                 if (rc < 0) {
 			dbug(3, "cpu=%d poll=%d errno=%d\n", cpu, rc, errno);
@@ -164,25 +181,27 @@ static void *reader_thread(void *data)
 				goto error_out;
                         }
                 }
+		dbug(3, "thread %d end ppoll\n", cpu);
+		if (switch_file[cpu]) {
+			switch_file[cpu] = 0;
+			if (switch_outfile(cpu, &fnum) < 0)
+				goto error_out;
+			wsize = 0;
+		}
+
 		while ((rc = read(relay_fd[cpu], buf, sizeof(buf))) > 0) {
-			wsize += rc;
 			/* Switching file */
-			if (fsize_max && wsize > fsize_max) {
-				close(out_fd[cpu]);
-				fnum++;
-				if (fnum_max && fnum == fnum_max)
-					remove_file = 1;
-				if (open_outfile(fnum, cpu, remove_file) < 0) {
-					perr("Couldn't open file for cpu %d, exiting.", cpu);
+			if (fsize_max && wsize + rc > fsize_max) {
+				if (switch_outfile(cpu, &fnum) < 0)
 					goto error_out;
-				}
-				wsize = rc;
+				wsize = 0;
 			}
 			if (write(out_fd[cpu], buf, rc) != rc) {
 				if (errno != EPIPE)
 					perr("Couldn't write to output %d for cpu %d, exiting.", out_fd[cpu], cpu);
 				goto error_out;
 			}
+			wsize += rc;
 		}
         } while (!stop_threads);
 	dbug(3, "exiting thread for cpu %d\n", cpu);
@@ -195,6 +214,19 @@ error_out:
 	return(NULL);
 }
 
+static void switchfile_handler(int sig)
+{
+	int i;
+	dbug(3, "file switching signal %d received\n", sig);
+	for (i = 0; i < ncpus; i++) {
+		if (reader[i]) {
+			switch_file[i] = 1;
+			pthread_kill(reader[i], SIGUSR2);
+		} else
+			break;
+	}
+}
+
 /**
  *	init_relayfs - create files and threads for relayfs processing
  *
@@ -308,6 +340,12 @@ int init_relayfs(void)
 		
 	}
 	if (!load_only) {
+		struct sigaction sa;
+
+		sa.sa_handler = switchfile_handler;
+		sa.sa_flags = 0;
+		sigemptyset(&sa.sa_mask);
+		sigaction(SIGUSR1, &sa, NULL);
 		dbug(2, "starting threads\n");
 		for (i = 0; i < ncpus; i++) {
 			if (pthread_create(&reader[i], NULL, reader_thread,


-- 
Masami Hiramatsu

Software Engineer
Hitachi Computer Products (America), Inc.
Software Solutions Division

e-mail: mhiramat@redhat.com

Re: [PATCH 0/3] Signal-based file switching support

[Masami Hiramatsu]

Masami Hiramatsu wrote:
> Hi,
>
> This patch series adds signal-based file switching support on systemtap
> runtime. This feature allows administators to move their stap logs to
> other storage safely on file-flight recorder mode (-F and -o option
> specified).

No comment?
If there is no claim, I'd like to commit it.

Thank you,

-- 
Masami Hiramatsu

Software Engineer
Hitachi Computer Products (America), Inc.
Software Solutions Division

e-mail: mhiramat@redhat.com