opensync: add kernel crash detection

This feature makes use of the crashlog driver in the kernel. If we experience
an oops, the kernel will store the stack trace inside /sys/kernel/debug/crashlog.
OpenSync will then add an entry to the Alarms table. The could can then retrieve
the crashlog via the Command_config table.

A crash can be simulated with
   echo c > /proc/sysrq-trigger

Signed-off-by: John Crispin <john@phrozen.org>

feeds/wlan-ap/opensync/files/bin/wlan_ap_crashlog.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+/usr/opensync/tools/ovsh insert Command_Config command:="crashlog" delay:=5 timestamp:=123 payload:="[\"map\",[[\"ul_url\",\"http://192.168.178.9/upload\"]]]"

feeds/wlan-ap/opensync/src/platform/openwrt/src/command/inc/command.h

@@ -35,6 +35,7 @@ struct task {
 };
 
 extern char serial[64];
+extern time_t crash_timestamp;
 extern pid_t cmd_handler_tcpdump(struct task *task);
 extern void task_status(struct task *task, int status, char *result);
 extern void task_init(void);
@@ -42,5 +43,7 @@ extern int ubus_get_l3_device(const char *net, char *ifname);
 extern int command_ubus_init(struct ev_loop *loop);
 extern void node_config_init(void);
 extern void webserver_init(void);
+extern void crashlog_init(void);
+extern pid_t cmd_handler_crashlog(struct task *task);
 
 #endif

feeds/wlan-ap/opensync/src/platform/openwrt/src/command/src/cmd_crashlog.c

@@ -0,0 +1,63 @@
+#include <libubox/list.h>
+#include <evsched.h>
+#include <net/if.h>
+
+#include "command.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+static void cmd_handler_crashlog_cb(struct ev_loop *loop, ev_child *child, int revents)
+{
+	struct task *task = container_of(child, struct task, child);
+
+	ev_child_stop(loop, child);
+	if (child->rstatus) {
+		task_status(task, TASK_FAILED, "upload failed");
+		return;
+	}
+	task_status(task, TASK_COMPLETE, NULL);
+}
+
+pid_t cmd_handler_crashlog(struct task *task)
+{
+
+	char *_url = NULL;
+	char url[128];
+	char *argv[] = { "/usr/bin/curl", "-T", "/tmp/crashlog", url, NULL };
+	pid_t pid;
+
+	if (!crash_timestamp) {
+		task_status(task, TASK_FAILED, "no crashlog found");
+		return -1;
+	}
+
+	system("cp /sys/kernel/debug/crashlog /tmp/crashlog");
+
+	_url = SCHEMA_KEY_VAL(task->conf.payload, "ul_url");
+	if (!_url) {
+		LOG(ERR, "curl: command without a valid ul_url");
+		task_status(task, TASK_FAILED, "invalid url");
+		return -1;
+	}
+	snprintf(url, sizeof(url), "%s/%s-%lu.crashlog", _url, serial, crash_timestamp);
+
+	pid = fork();
+	if (pid == 0) {
+		execv(*argv, argv);
+		LOG(ERR, "curl: failed to start");
+		exit(1);
+	}
+
+	if (pid < 0) {
+		LOG(ERR, "curl, failed to fork");
+		task_status(task, TASK_FAILED, "forking curl failed");
+		return -1;
+	}
+
+	ev_child_init(&task->child, cmd_handler_crashlog_cb, pid, 0);
+	ev_child_start(EV_DEFAULT, &task->child);
+
+	LOGN("curl: started");
+	return pid;
+}

feeds/wlan-ap/opensync/src/platform/openwrt/src/command/src/crashlog.c

@@ -0,0 +1,25 @@
+#include "command.h"
+
+static ovsdb_table_t table_Alarms;
+time_t crash_timestamp = 0;
+
+void crashlog_init(void)
+{
+	struct schema_Alarms alarm = {};
+	struct stat s = {};
+
+	if (stat("/sys/kernel/debug/crashlog", &s) < 0)
+		return;
+
+	crash_timestamp = time(NULL);
+	OVSDB_TABLE_INIT_NO_KEY(Alarms);
+
+	SCHEMA_SET_STR(alarm.add_info, "reboot");
+	SCHEMA_SET_STR(alarm.code, "oops");
+	SCHEMA_SET_STR(alarm.source, "kernel");
+	SCHEMA_SET_INT(alarm.timestamp, crash_timestamp);
+	if (!ovsdb_table_upsert(&table_Alarms, &alarm, false))
+		LOG(ERR, "crashlog: failed to insert alarm");
+
+	return;
+}

feeds/wlan-ap/opensync/src/platform/openwrt/src/command/src/main.c

@@ -80,6 +80,7 @@ int main(int argc, char ** argv)
 	node_config_init();
 	command_ubus_init(loop);
 	webserver_init();
+	crashlog_init();
 
 	ev_run(loop, 0);
 

feeds/wlan-ap/opensync/src/platform/openwrt/src/command/src/task.c

@@ -15,6 +15,9 @@ static struct cmd_handler {
 	{
 		.cmd = "tcpdump",
 		.cb = cmd_handler_tcpdump,
+	}, {
+		.cmd = "crashlog",
+		.cb = cmd_handler_crashlog,
 	},
 };
 

feeds/wlan-ap/opensync/src/platform/openwrt/src/command/unit.mk

@@ -40,6 +40,8 @@ UNIT_SRC    += src/task.c
 UNIT_SRC    += src/node_config.c
 UNIT_SRC    += src/cmd_tcpdump.c
 UNIT_SRC    += src/webserver.c
+UNIT_SRC    += src/crashlog.c
+UNIT_SRC    += src/cmd_crashlog.c
 
 UNIT_CFLAGS := -I$(UNIT_PATH)/inc
 UNIT_CFLAGS += -Isrc/lib/common/inc/

patches/pending/0004-kernel-fix-crashlog-for-ARM.patch

@@ -0,0 +1,144 @@
+From 3fe28d6dbd80231937cb5569cc8ed28fb51b9d28 Mon Sep 17 00:00:00 2001
+From: John Crispin <john@phrozen.org>
+Date: Fri, 24 Jul 2020 11:55:31 +0200
+Subject: [PATCH] kernel: fix crashlog for ARM
+
+The current driver was MIPS specific, make it work on ARM.
+
+Signed-off-by: John Crispin <john@phrozen.org>
+---
+ config/Config-kernel.in                       |  2 +-
+ .../generic/hack-4.14/930-crashlog.patch      | 52 +++++++++++--------
+ 2 files changed, 30 insertions(+), 24 deletions(-)
+
+diff --git a/config/Config-kernel.in b/config/Config-kernel.in
+index 87053b7f23..eb61db542f 100644
+--- a/config/Config-kernel.in
++++ b/config/Config-kernel.in
+@@ -28,7 +28,7 @@ config KERNEL_PRINTK
+ 
+ config KERNEL_CRASHLOG
+ 	bool "Crash logging"
+-	depends on !(arm || powerpc || sparc || TARGET_uml || i386 || x86_64)
++	depends on !(powerpc || sparc || TARGET_uml || i386 || x86_64)
+ 	default y
+ 
+ config KERNEL_SWAP
+diff --git a/target/linux/generic/hack-4.14/930-crashlog.patch b/target/linux/generic/hack-4.14/930-crashlog.patch
+index 9d09dbd760..0489e0ef41 100644
+--- a/target/linux/generic/hack-4.14/930-crashlog.patch
++++ b/target/linux/generic/hack-4.14/930-crashlog.patch
+@@ -19,8 +19,10 @@ Signed-off-by: Felix Fietkau <nbd@nbd.name>
+  create mode 100644 include/linux/crashlog.h
+  create mode 100644 kernel/crashlog.c
+ 
++Index: linux-4.14.187/include/linux/crashlog.h
++===================================================================
+ --- /dev/null
+-+++ b/include/linux/crashlog.h
+++++ linux-4.14.187/include/linux/crashlog.h
+ @@ -0,0 +1,17 @@
+ +#ifndef __CRASHLOG_H
+ +#define __CRASHLOG_H
+@@ -39,8 +41,10 @@ Signed-off-by: Felix Fietkau <nbd@nbd.name>
+ +#endif
+ +
+ +#endif
+---- a/init/Kconfig
+-+++ b/init/Kconfig
++Index: linux-4.14.187/init/Kconfig
++===================================================================
++--- linux-4.14.187.orig/init/Kconfig
+++++ linux-4.14.187/init/Kconfig
+ @@ -1009,6 +1009,10 @@ config RELAY
+  
+  	  If unsure, say N.
+@@ -52,8 +56,10 @@ Signed-off-by: Felix Fietkau <nbd@nbd.name>
+  config BLK_DEV_INITRD
+  	bool "Initial RAM filesystem and RAM disk (initramfs/initrd) support"
+  	depends on BROKEN || !FRV
+---- a/kernel/Makefile
+-+++ b/kernel/Makefile
++Index: linux-4.14.187/kernel/Makefile
++===================================================================
++--- linux-4.14.187.orig/kernel/Makefile
+++++ linux-4.14.187/kernel/Makefile
+ @@ -110,6 +110,7 @@ obj-$(CONFIG_CONTEXT_TRACKING) += contex
+  obj-$(CONFIG_TORTURE_TEST) += torture.o
+  
+@@ -62,9 +68,11 @@ Signed-off-by: Felix Fietkau <nbd@nbd.name>
+  
+  $(obj)/configs.o: $(obj)/config_data.h
+  
++Index: linux-4.14.187/kernel/crashlog.c
++===================================================================
+ --- /dev/null
+-+++ b/kernel/crashlog.c
+-@@ -0,0 +1,213 @@
+++++ linux-4.14.187/kernel/crashlog.c
++@@ -0,0 +1,205 @@
+ +/*
+ + * Crash information logger
+ + * Copyright (C) 2010 Felix Fietkau <nbd@nbd.name>
+@@ -131,17 +139,9 @@ Signed-off-by: Felix Fietkau <nbd@nbd.name>
+ +
+ +	if (crashlog_addr)
+ +		return false;
+-+
+-+	if (addr > limit)
+++	if (size <= limit)
+ +		return false;
+-+
+-+	if (addr + size > limit)
+-+		size = limit - addr;
+-+
+-+	crashlog_addr = addr;
+-+
+-+	if (addr + size > CRASHLOG_OFFSET)
+-+		crashlog_addr += size - CRASHLOG_OFFSET;
+++	crashlog_addr = addr + limit - CRASHLOG_OFFSET;
+ +
+ +	return true;
+ +}
+@@ -278,8 +278,10 @@ Signed-off-by: Felix Fietkau <nbd@nbd.name>
+ +	return 0;
+ +}
+ +module_init(crashlog_init_fs);
+---- a/kernel/module.c
+-+++ b/kernel/module.c
++Index: linux-4.14.187/kernel/module.c
++===================================================================
++--- linux-4.14.187.orig/kernel/module.c
+++++ linux-4.14.187/kernel/module.c
+ @@ -256,6 +256,9 @@ static void mod_update_bounds(struct mod
+  #ifdef CONFIG_KGDB_KDB
+  struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */
+@@ -290,8 +292,10 @@ Signed-off-by: Felix Fietkau <nbd@nbd.name>
+  
+  static void module_assert_mutex(void)
+  {
+---- a/mm/bootmem.c
+-+++ b/mm/bootmem.c
++Index: linux-4.14.187/mm/bootmem.c
++===================================================================
++--- linux-4.14.187.orig/mm/bootmem.c
+++++ linux-4.14.187/mm/bootmem.c
+ @@ -15,6 +15,7 @@
+  #include <linux/export.h>
+  #include <linux/kmemleak.h>
+@@ -308,8 +312,10 @@ Signed-off-by: Felix Fietkau <nbd@nbd.name>
+  	map = bdata->node_bootmem_map;
+  	start = bdata->node_min_pfn;
+  	end = bdata->node_low_pfn;
+---- a/mm/memblock.c
+-+++ b/mm/memblock.c
++Index: linux-4.14.187/mm/memblock.c
++===================================================================
++--- linux-4.14.187.orig/mm/memblock.c
+++++ linux-4.14.187/mm/memblock.c
+ @@ -19,6 +19,7 @@
+  #include <linux/debugfs.h>
+  #include <linux/seq_file.h>
+-- 
+2.25.1
+