Merge pull request #131 from BrainWart/tests/native-posix-test-exploration

Native posix integration testing

app/boards/native_posix.conf

@@ -1,9 +1,8 @@
 CONFIG_KSCAN=n
 CONFIG_ZMK_KSCAN_MOCK_DRIVER=y
-CONFIG_ZMK_KSCAN_COMPOSITE_DRIVER=y
 CONFIG_ZMK_KSCAN_GPIO_DRIVER=n
 CONFIG_GPIO=n
 CONFIG_ZMK_BLE=n
 CONFIG_LOG=y
-CONFIG_KSCAN_LOG_LEVEL_DBG=y
+CONFIG_LOG_BACKEND_SHOW_COLOR=n
 CONFIG_ZMK_LOG_LEVEL_DBG=y

app/boards/native_posix.overlay

@@ -1,79 +1,18 @@
-#include <dt-bindings/zmk/keys.h>
 #include <behaviors.dtsi>
-#include <zmk/kscan-mock.h>
+#include <dt-bindings/zmk/keys.h>
+#include <dt-bindings/zmk/kscan-mock.h>
 
 / {
 	chosen {
-		zmk,kscan = &kscan0;
+		zmk,kscan = &kscan;
-		zmk,keymap = &keymap0;
 	};
 
-	kscan0: kscan_0 {
+	kscan: kscan {
-		compatible = "zmk,kscan-composite";
-		label = "KSCAN_COMP";
-		rows = <2>;
-		columns = <4>;
-
-		left: left {
-			kscan = <&left_hand>;
-		};
-
-		right: right {
-			kscan = <&right_hand>;
-			column-offset = <2>;
-		};
-	};
-
-	left_hand: kscan_1 {
 		compatible = "zmk,kscan-mock";
-		label = "KSCAN_LEFT";
+		label = "KSCAN_MOCK";
 
 		rows = <2>;
 		columns = <2>;
-		events = <ZMK_MOCK_PRESS(0,1,300) ZMK_MOCK_PRESS(0,0,300) ZMK_MOCK_RELEASE(0,0,300) ZMK_MOCK_RELEASE(0,1,300)>;
+		exit-after;
-		// events = <ZMK_MOCK_PRESS(0,0,800) ZMK_MOCK_RELEASE(0,0,800) ZMK_MOCK_PRESS(0,1,800) ZMK_MOCK_RELEASE(0,1,800)>;
-	};
-
-	right_hand: kscan_2 {
-		compatible = "zmk,kscan-mock";
-		label = "KSCAN_RIGHT";
-
-		rows = <2>;
-		columns = <2>;
-		events = <ZMK_MOCK_PRESS(1,1,800) ZMK_MOCK_RELEASE(1,1,100) ZMK_MOCK_PRESS(0,1,800) ZMK_MOCK_RELEASE(0,1,100)>;
-	};
-
-	keymap0: keymap {
-		compatible = "zmk,keymap";
-		label ="Default keymap";
-		layers = <&default &lower &raise>;
-	};
-
-	layers {
-		compatible = "zmk,layers";
-
-		default: layer_0 {
-			label = "DEFAULT";
-
-			bindings = <
-				&kp B &mo 1 &kp C &kp E
-				&kp D &kp G &kp F &kp Q>;
-		};
-
-		lower: layer_1 {
-			label = "LOWER";
-
-			bindings = <
-				&cp M_NEXT &trans &kp O &kp Q
-				&kp L  &kp J &kp P &kp K>;
-		};
-
-		raise: layer_2 {
-			label = "RAISE";
-
-			bindings = <
-				&kp W &kp U &kp I &kp N
-				&kp X  &kp M &kp C &kp B>;
-		};
 	};
 };

app/dts/bindings/zmk,kscan-mock.yaml

@@ -15,3 +15,5 @@ properties:
     type: int
   columns:
     type: int
+  exit-after:
+    type: boolean

app/run-test.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+#
+# Copyright (c) 2020 Peter Johanson; Cody McGinnis
+#
+# SPDX-License-Identifier: MIT
+#
+
+if [ -z "$1" ]; then
+	echo "Usage: ./run-test.sh <path to testcase>"
+	exit 1
+elif [ "$1" = "all" ]; then
+	echo "" > ./build/tests/pass-fail.log
+	find tests -name native_posix.keymap -exec dirname \{\} \; | xargs -l -P 4 ./run-test.sh
+	err=$?
+	sort -k2 ./build/tests/pass-fail.log
+	exit $err
+fi
+
+testcase="$1"
+echo "Running $testcase:"
+
+west build -d build/$testcase -b native_posix -- -DZMK_CONFIG=$testcase > /dev/null 2>&1
+if [ $? -gt 0 ]; then
+	echo "FAIL: $testcase did not build" >> ./build/tests/pass-fail.log
+else
+	./build/$testcase/zephyr/zmk.exe | sed -e "s/.*> //" | tee build/$testcase/keycode_events_full.log | sed -n -f $testcase/events.patterns > build/$testcase/keycode_events.log
+	diff -au $testcase/keycode_events.snapshot build/$testcase/keycode_events.log
+	if [ $? -gt 0 ]; then
+		if [ -f $testcase/pending ]; then
+			echo "PEND: $testcase" >> ./build/tests/pass-fail.log
+			exit 0
+		else
+			echo "FAIL: $testcase" >> ./build/tests/pass-fail.log
+			exit 1
+		fi
+	else
+		echo "PASS: $testcase" >> ./build/tests/pass-fail.log
+		exit 0
+	fi
+fi

app/src/behaviors/behavior_toggle_layer.c

@@ -14,22 +14,29 @@
 
 LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL);
 
-struct behavior_tog_config { };
+struct behavior_tog_config
-struct behavior_tog_data { };
+{
+};
+struct behavior_tog_data
+{
+};
 
 static int behavior_tog_init(struct device *dev)
 {
   return 0;
 };
 
-
 static int tog_keymap_binding_pressed(struct device *dev, u32_t position, u32_t layer, u32_t _)
 {
+  LOG_DBG("position %d layer %d", position, layer);
+
   return zmk_keymap_layer_toggle(layer);
 }
 
 static int tog_keymap_binding_released(struct device *dev, u32_t position, u32_t layer, u32_t _)
 {
+  LOG_DBG("position %d layer %d", position, layer);
+
   return 0;
 }
 

app/src/kscan_mock.c

@@ -6,13 +6,14 @@
 
 #define DT_DRV_COMPAT zmk_kscan_mock
 
+#include <stdlib.h>
 #include <device.h>
 #include <drivers/kscan.h>
 #include <logging/log.h>
 
 LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL);
 
-#include <zmk/kscan-mock.h>
+#include <dt-bindings/zmk/kscan-mock.h>
 
 struct kscan_mock_data
 {
@@ -50,6 +51,7 @@ static int kscan_mock_configure(struct device *dev, kscan_callback_t callback)
     struct kscan_mock_config_##n                                                   \
     {                                                                              \
         u32_t events[DT_INST_PROP_LEN(n, events)];                                 \
+        bool exit_after;                                                           \
     };                                                                             \
     static void kscan_mock_schedule_next_event_##n(struct device *dev)             \
     {                                                                              \
@@ -60,6 +62,9 @@ static int kscan_mock_configure(struct device *dev, kscan_callback_t callback)
             u32_t ev = cfg->events[data->event_index];                             \
             LOG_DBG("delaying next keypress: %d", ZMK_MOCK_MSEC(ev));              \
             k_delayed_work_submit(&data->work, K_MSEC(ZMK_MOCK_MSEC(ev)));         \
+        } else if (cfg->exit_after) {                                              \
+            LOG_DBG("Exiting");                                                    \
+            exit(0);                                                               \
         }                                                                          \
     }                                                                              \
     static void kscan_mock_work_handler_##n(struct k_work *work)                   \
@@ -67,12 +72,13 @@ static int kscan_mock_configure(struct device *dev, kscan_callback_t callback)
         struct kscan_mock_data *data =                                             \
             CONTAINER_OF(work, struct kscan_mock_data, work);                      \
         const struct kscan_mock_config_##n *cfg = data->dev->config_info;          \
-        u32_t ev = cfg->events[data->event_index++];                               \
+        u32_t ev = cfg->events[data->event_index];                                 \
         LOG_DBG("ev %u row %d column %d state %d\n", ev,                           \
                 ZMK_MOCK_ROW(ev), ZMK_MOCK_COL(ev), ZMK_MOCK_IS_PRESS(ev));        \
         data->callback(data->dev,                                                  \
                        ZMK_MOCK_ROW(ev), ZMK_MOCK_COL(ev), ZMK_MOCK_IS_PRESS(ev)); \
         kscan_mock_schedule_next_event_##n(data->dev);                             \
+        data->event_index++;                                                       \
     }                                                                              \
     static int kscan_mock_init_##n(struct device *dev)                             \
     {                                                                              \
@@ -93,7 +99,8 @@ static int kscan_mock_configure(struct device *dev, kscan_callback_t callback)
     };                                                                             \
     static struct kscan_mock_data kscan_mock_data_##n;                             \
     static const struct kscan_mock_config_##n kscan_mock_config_##n = {            \
-        .events = DT_INST_PROP(n, events)};                                        \
+        .events = DT_INST_PROP(n, events),                                         \
+        .exit_after = DT_INST_PROP(n, exit_after) };                               \
     DEVICE_AND_API_INIT(kscan_mock_##n, DT_INST_LABEL(n), kscan_mock_init_##n,     \
                         &kscan_mock_data_##n,                                      \
                         &kscan_mock_config_##n,                                    \

app/tests/keypress/behavior_keymap.dtsi

@@ -0,0 +1,28 @@
+#include <dt-bindings/zmk/keys.h>
+#include <behaviors.dtsi>
+#include <dt-bindings/zmk/kscan-mock.h>
+
+/ {
+	keymap {
+		compatible = "zmk,keymap";
+		label ="Default keymap";
+
+		default_layer {
+			bindings = <
+				&kp B &mo 1
+				&kp D &kp G>;
+		};
+
+		lower_layer {
+			bindings = <
+				&cp M_NEXT &trans
+				&kp L  &kp J>;
+		};
+
+		raise_layer {
+			bindings = <
+				&kp W &kp U
+				&kp X  &kp M>;
+		};
+	};
+};

app/tests/keypress/cp-press-release/events.patterns

@@ -0,0 +1 @@
+s/.*hid_listener_keycode_//p

app/tests/keypress/cp-press-release/keycode_events.snapshot

@@ -0,0 +1,2 @@
+pressed: usage_page 0x0c keycode 0xb5
+released: usage_page 0x0c keycode 0xb5

app/tests/keypress/cp-press-release/native_posix.keymap

@@ -0,0 +1,5 @@
+#include "../behavior_keymap.dtsi"
+
+&kscan {
+	events = <ZMK_MOCK_PRESS(0,1,10) ZMK_MOCK_PRESS(0,0,10) ZMK_MOCK_RELEASE(0,0,10) ZMK_MOCK_RELEASE(0,1,10)>;
+};

app/tests/keypress/kp-press-release/events.patterns

@@ -0,0 +1 @@
+s/.*hid_listener_keycode_//p

app/tests/keypress/kp-press-release/keycode_events.snapshot

@@ -0,0 +1,2 @@
+pressed: usage_page 0x0c keycode 0xb5
+released: usage_page 0x0c keycode 0xb5

app/tests/keypress/kp-press-release/native_posix.keymap

@@ -0,0 +1,8 @@
+#include <dt-bindings/zmk/keys.h>
+#include <behaviors.dtsi>
+#include <dt-bindings/zmk/kscan-mock.h>
+#include "../behavior_keymap.dtsi"
+
+&kscan {
+	events = <ZMK_MOCK_PRESS(0,1,10) ZMK_MOCK_PRESS(0,0,10) ZMK_MOCK_RELEASE(0,0,10) ZMK_MOCK_RELEASE(0,1,10)>;
+};

app/tests/momentary-layer/behavior_keymap.dtsi

@@ -0,0 +1,28 @@
+#include <dt-bindings/zmk/keys.h>
+#include <behaviors.dtsi>
+#include <dt-bindings/zmk/kscan-mock.h>
+
+/ {
+	keymap {
+		compatible = "zmk,keymap";
+		label ="Default keymap";
+
+		default_layer {
+			bindings = <
+				&kp B &mo 1
+				&kp D &kp G>;
+		};
+
+		lower_layer {
+			bindings = <
+				&cp M_NEXT &trans
+				&kp L  &kp J>;
+		};
+
+		raise_layer {
+			bindings = <
+				&kp W &kp U
+				&kp X  &kp M>;
+		};
+	};
+};

app/tests/momentary-layer/early-key-release/events.patterns

@@ -0,0 +1,2 @@
+s/.*hid_listener_keycode/kp/p
+s/.*mo_keymap_binding/mo/p

app/tests/momentary-layer/early-key-release/keycode_events.snapshot

@@ -0,0 +1,4 @@
+kp_pressed: usage_page 0x07 keycode 0x05
+mo_pressed: position 1 layer 1
+kp_released: usage_page 0x07 keycode 0x05
+mo_released: position 1 layer 1

app/tests/momentary-layer/early-key-release/native_posix.keymap

@@ -0,0 +1,8 @@
+#include <dt-bindings/zmk/keys.h>
+#include <behaviors.dtsi>
+#include <dt-bindings/zmk/kscan-mock.h>
+#include "../behavior_keymap.dtsi"
+
+&kscan {
+	events = <ZMK_MOCK_PRESS(0,0,10) ZMK_MOCK_PRESS(0,1,10) ZMK_MOCK_RELEASE(0,0,10) ZMK_MOCK_RELEASE(0,1,10)>;
+};

app/tests/momentary-layer/normal/events.patterns

@@ -0,0 +1,2 @@
+s/.*hid_listener_keycode/kp/p
+s/.*mo_keymap_binding/mo/p

app/tests/momentary-layer/normal/keycode_events.snapshot

@@ -0,0 +1,4 @@
+mo_pressed: position 1 layer 1
+kp_pressed: usage_page 0x0c keycode 0xb5
+kp_released: usage_page 0x0c keycode 0xb5
+mo_released: position 1 layer 1

app/tests/momentary-layer/normal/native_posix.keymap

@@ -0,0 +1,8 @@
+#include <dt-bindings/zmk/keys.h>
+#include <behaviors.dtsi>
+#include <dt-bindings/zmk/kscan-mock.h>
+#include "../behavior_keymap.dtsi"
+
+&kscan {
+	events = <ZMK_MOCK_PRESS(0,1,10) ZMK_MOCK_PRESS(0,0,10) ZMK_MOCK_RELEASE(0,0,10) ZMK_MOCK_RELEASE(0,1,10)>;
+};

app/tests/none/behavior_keymap.dtsi

@@ -0,0 +1,22 @@
+#include <dt-bindings/zmk/keys.h>
+#include <behaviors.dtsi>
+#include <dt-bindings/zmk/kscan-mock.h>
+
+/ {
+	keymap {
+		compatible = "zmk,keymap";
+		label ="Default keymap";
+
+		default_layer {
+			bindings = <
+				&none &mo 1
+				&kp A &none>;
+		};
+
+		lower_layer {
+			bindings = <
+				&none &trans
+				&none  &kp A>;
+		};
+	};
+};

app/tests/none/layered/events.patterns

@@ -0,0 +1 @@
+s/.*hid_listener_keycode/kp/p

app/tests/none/layered/native_posix.keymap

@@ -0,0 +1,8 @@
+#include <dt-bindings/zmk/keys.h>
+#include <behaviors.dtsi>
+#include <dt-bindings/zmk/kscan-mock.h>
+#include "../behavior_keymap.dtsi"
+
+&kscan {
+	events = <ZMK_MOCK_PRESS(0,1,10) ZMK_MOCK_PRESS(1,0,10) ZMK_MOCK_RELEASE(1,0,10) ZMK_MOCK_RELEASE(0,1,10)>;
+};

app/tests/none/normal/events.patterns

@@ -0,0 +1 @@
+s/.*hid_listener_keycode/kp/p

app/tests/none/normal/native_posix.keymap

@@ -0,0 +1,8 @@
+#include <dt-bindings/zmk/keys.h>
+#include <behaviors.dtsi>
+#include <dt-bindings/zmk/kscan-mock.h>
+#include "../behavior_keymap.dtsi"
+
+&kscan {
+	events = <ZMK_MOCK_PRESS(0,0,10) ZMK_MOCK_RELEASE(0,0,10)>;
+};

app/tests/toggle-layer/behavior_keymap.dtsi

@@ -0,0 +1,28 @@
+#include <dt-bindings/zmk/keys.h>
+#include <behaviors.dtsi>
+#include <dt-bindings/zmk/kscan-mock.h>
+
+/ {
+	keymap {
+		compatible = "zmk,keymap";
+		label ="Default keymap";
+
+		default_layer {
+			bindings = <
+				&kp B &tog 1
+				&kp D &kp G>;
+		};
+
+		lower_layer {
+			bindings = <
+				&cp M_NEXT &trans
+				&kp L  &kp J>;
+		};
+
+		raise_layer {
+			bindings = <
+				&kp W &kp U
+				&kp X  &kp M>;
+		};
+	};
+};

app/tests/toggle-layer/early-key-release/events.patterns

@@ -0,0 +1,2 @@
+s/.*hid_listener_keycode/kp/p
+s/.*tog_keymap_binding/tog/p

app/tests/toggle-layer/early-key-release/keycode_events.snapshot

@@ -0,0 +1,6 @@
+kp_pressed: usage_page 0x07 keycode 0x05
+tog_pressed: position 1 layer 1
+kp_released: usage_page 0x07 keycode 0x05
+tog_released: position 1 layer 1
+kp_pressed: usage_page 0x0c keycode 0xb5
+kp_released: usage_page 0x0c keycode 0xb5

app/tests/toggle-layer/early-key-release/native_posix.keymap

@@ -0,0 +1,9 @@
+#include <dt-bindings/zmk/keys.h>
+#include <behaviors.dtsi>
+#include <dt-bindings/zmk/kscan-mock.h>
+#include "../behavior_keymap.dtsi"
+
+&kscan {
+	events = <ZMK_MOCK_PRESS(0,0,10) ZMK_MOCK_PRESS(0,1,10) ZMK_MOCK_RELEASE(0,0,10) ZMK_MOCK_RELEASE(0,1,10)
+		ZMK_MOCK_PRESS(0,0,10) ZMK_MOCK_RELEASE(0,0,10)>;
+};

app/tests/toggle-layer/normal/events.patterns

@@ -0,0 +1,2 @@
+s/.*hid_listener_keycode/kp/p
+s/.*tog_keymap_binding/tog/p

app/tests/toggle-layer/normal/keycode_events.snapshot

@@ -0,0 +1,4 @@
+tog_pressed: position 1 layer 1
+tog_released: position 1 layer 1
+kp_pressed: usage_page 0x0c keycode 0xb5
+kp_released: usage_page 0x0c keycode 0xb5

app/tests/toggle-layer/normal/native_posix.keymap

@@ -0,0 +1,8 @@
+#include <dt-bindings/zmk/keys.h>
+#include <behaviors.dtsi>
+#include <dt-bindings/zmk/kscan-mock.h>
+#include "../behavior_keymap.dtsi"
+
+&kscan {
+	events = <ZMK_MOCK_PRESS(0,1,10) ZMK_MOCK_RELEASE(0,1,10) ZMK_MOCK_PRESS(0,0,10) ZMK_MOCK_RELEASE(0,0,10)>;
+};

app/tests/transparent/behavior_keymap.dtsi

@@ -0,0 +1,22 @@
+#include <dt-bindings/zmk/keys.h>
+#include <behaviors.dtsi>
+#include <dt-bindings/zmk/kscan-mock.h>
+
+/ {
+	keymap {
+		compatible = "zmk,keymap";
+		label ="Default keymap";
+
+		default_layer {
+			bindings = <
+				&trans &mo 1
+				&kp A &none>;
+		};
+
+		lower_layer {
+			bindings = <
+				&trans &trans
+				&trans  &kp A>;
+		};
+	};
+};

app/tests/transparent/layered/events.patterns

@@ -0,0 +1 @@
+s/.*hid_listener_keycode/kp/p

app/tests/transparent/layered/keycode_events.snapshot

@@ -0,0 +1,2 @@
+kp_pressed: usage_page 0x07 keycode 0x04
+kp_released: usage_page 0x07 keycode 0x04

app/tests/transparent/layered/native_posix.keymap

@@ -0,0 +1,8 @@
+#include <dt-bindings/zmk/keys.h>
+#include <behaviors.dtsi>
+#include <dt-bindings/zmk/kscan-mock.h>
+#include "../behavior_keymap.dtsi"
+
+&kscan {
+	events = <ZMK_MOCK_PRESS(0,1,10) ZMK_MOCK_PRESS(1,0,10) ZMK_MOCK_RELEASE(1,0,10) ZMK_MOCK_RELEASE(0,1,10)>;
+};

app/tests/transparent/normal/events.patterns

@@ -0,0 +1 @@
+s/.*hid_listener_keycode/kp/p

app/tests/transparent/normal/native_posix.keymap

@@ -0,0 +1,8 @@
+#include <dt-bindings/zmk/keys.h>
+#include <behaviors.dtsi>
+#include <dt-bindings/zmk/kscan-mock.h>
+#include "../behavior_keymap.dtsi"
+
+&kscan {
+	events = <ZMK_MOCK_PRESS(0,0,10) ZMK_MOCK_RELEASE(0,0,10)>;
+};

docs/docs/dev-tests.md

@@ -0,0 +1,20 @@
+---
+id: dev-tests
+title: Tests
+sidebar_label: Tests
+---
+
+Running tests requires [native posix support](./dev-posix-board). Any folder under `/app/tests`
+containing `native_posix.keymap` will be selected when running `./run-test.sh all`.
+
+## Creating a New Test Set
+1. Copy the test set that most closely resembles the tests you will be creating.
+2. Rename the newly created test set to the behavior you're testing  e.g, toggle-layer
+3. Modify `behavior_keymap.dtsi` to create a keymap using the behavior and related behaviors
+4. Modify `test_case/native_posix.keymap` for a simulated use case
+5. Modify `test_case/events.patterns` to collect relevant logs to the test
+    - See: [sed manual](https://www.gnu.org/software/sed/manual/sed.html) and
+    [tutorial](https://www.digitalocean.com/community/tutorials/the-basics-of-using-the-sed-stream-editor-to-manipulate-text-in-linux)
+6. Modify `test_case/keycode_events.snapshot` for to include the expected output
+7. Rename the `test_case` folder to describe the test.
+8. Repeat steps 4 to 7 for every test case

docs/sidebars.js

@@ -20,6 +20,7 @@ module.exports = {
       "dev-setup",
       "dev-boards-shields-keymaps",
       "dev-posix-board",
+      "dev-tests",
     ],
     "Dev Guides": ["dev-guide-new-shield", "dev-guide-usb-logging"],
   },