manifest: optional: move lz4 to external

Move lz4 to become external module. It is not in the default manifest
anymore (through submanifests) and will need to be added if application
requires it per the docs.

Samples will be moved to the module itself.

Signed-off-by: Anas Nashif <anas.nashif@intel.com>

MAINTAINERS.yml

@@ -6023,16 +6023,6 @@ West:
   labels:
     - "area: LVGL"
 
-"West project: lz4":
-  status: odd fixes
-  collaborators:
-    - parthitce
-  files:
-    - modules/lz4/
-    - samples/modules/compression/lz4/
-  labels:
-    - "area: Compression"
-
 "West project: mbedtls":
   status: maintained
   maintainers:

doc/develop/manifest/external/lz4.rst

@@ -0,0 +1,51 @@
+.. _external_module_lz4:
+
+LZ4 - Extremely fast compression
+################################
+
+Introduction
+************
+
+LZ4 is a lossless compression algorithm, providing compression speed greater
+than 500 MB/s per core, scalable with multi-core CPUs. It features an
+extremely fast decoder, with speed in multiple GB/s per core, typically
+reaching RAM speed limits on multi-core systems.
+
+Speed can be tuned dynamically by selecting an "acceleration" factor, which
+trades compression ratio for faster speed. On the other end, a high
+compression derivative, LZ4_HC, is also provided, trading CPU time for
+improved compression ratio. All versions feature the same decompression speed.
+
+LZ4 is also compatible with dictionary compression, both at API and CLI
+levels. It can ingest any input file as a dictionary, though only the final
+64KB are used. This capability can be combined
+
+
+Usage with Zephyr
+*****************
+
+To pull in lz4 as a Zephyr module, either add it as a West project in the ``west.yaml``
+file or pull it in by adding a submanifest (e.g. ``zephyr/submanifests/lz4.yaml``) file
+with the following content and run ``west update``:
+
+.. code-block:: yaml
+
+   manifest:
+     projects:
+       - name: lz4
+         url: https://github.com/zephyrproject-rtos/lz4
+         revision: zephyr
+         path: modules/lib/lz4 # adjust the path as needed
+
+For more detailed instructions and API documentation, refer to the `lz4 documentation`_ as
+well as the provided `lz4 examples`_.
+
+
+Reference
+*********
+
+.. _lz4 documentation:
+    https://github.com/lz4/lz4/tree/dev/doc
+
+.. _lz4 examples:
+   https://github.com/zephyrproject-rtos/lz4/tree/zephyr/zephyr/samples

modules/Kconfig

@@ -94,9 +94,6 @@ comment "Trusted-firmware-a module not available."
 comment "Nanopb module not available."
 	depends on !ZEPHYR_NANOPB_MODULE
 
-comment "Lz4 module not available."
-	depends on !ZEPHYR_LZ4_MODULE
-
 comment "loramac-node module not available."
 	depends on !ZEPHYR_LORAMAC_NODE_MODULE
 

modules/lz4/CMakeLists.txt

@@ -1,46 +0,0 @@
-# Copyright (c) 2020 Linumiz
-# SPDX-License-Identifier: Apache-2.0
-
-if(CONFIG_LZ4)
-  set(LZ4_DIR ${ZEPHYR_CURRENT_MODULE_DIR})
-
-  zephyr_library()
-
-  zephyr_include_directories(${LZ4_DIR}/lib)
-
-  zephyr_library_compile_definitions_ifdef(CONFIG_LZ4_HEAPMODE_STACK
-    LZ4_HEAPMODE=0
-  )
-
-  zephyr_library_compile_definitions_ifdef(CONFIG_LZ4_DISABLE_DYNAMIC_MEMORY_ALLOCATION
-    LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION
-  )
-
-  zephyr_library_compile_definitions(
-    LZ4_MEMORY_USAGE=${CONFIG_LZ4_MEMORY_USAGE}
-  )
-
-  zephyr_library_sources(
-    ${LZ4_DIR}/lib/lz4.c
-  )
-
-  zephyr_library_sources_ifdef(CONFIG_LZ4_HIGH_COMPRESSION_VARIANT
-    ${LZ4_DIR}/lib/lz4hc.c
-  )
-
-  zephyr_library_compile_definitions_ifdef(CONFIG_LZ4HC_HEAPMODE_STACK
-    LZ4HC_HEAPMODE=0
-  )
-
-  zephyr_library_sources_ifdef(CONFIG_LZ4_XX_HASH
-    ${LZ4_DIR}/lib/xxhash.c
-  )
-
-  zephyr_library_sources_ifdef(CONFIG_LZ4_FRAME_SUPPORT
-    ${LZ4_DIR}/lib/lz4frame.c
-  )
-
-  zephyr_library_compile_definitions_ifdef(CONFIG_LZ4F_HEAPMODE_HEAP
-    LZ4F_HEAPMODE=1
-  )
-endif()

modules/lz4/Kconfig

@@ -1,102 +0,0 @@
-# Copyright (c) 2020 Linumiz
-# SPDX-License-Identifier: Apache-2.0
-
-config ZEPHYR_LZ4_MODULE
-	bool
-
-menuconfig LZ4
-	bool "Lz4 data compression and decompression"
-	help
-	  This option enables lz4 compression & decompression library
-	  support.
-if LZ4
-
-config LZ4_MEMORY_USAGE
-	int "Lz4 memory usage"
-	range 10 20
-	default 14
-	help
-	  Increasing memory usage improves compression ratio, but usually at the
-	  cost of speed, due to cache locality. Memory usage 2^value (10 -> 1KB,
-	  12 -> 4KB, 20 -> 1MB).
-
-config LZ4_DISABLE_DYNAMIC_MEMORY_ALLOCATION
-	bool "Disable dynamic memory allocation"
-	help
-	  Disable lz4 functions that use dynamic memory allocation functions.
-
-choice LZ4_HEAPMODE
-	prompt "How stateless compression functions allocate memory for their hash table"
-	default LZ4_HEAPMODE_HEAP
-
-config LZ4_HEAPMODE_STACK
-	bool "in memory stack"
-	help
-	  Allocate memory from stack (fastest).
-
-config LZ4_HEAPMODE_HEAP
-	bool "in memory heap"
-	depends on !LZ4_DISABLE_DYNAMIC_MEMORY_ALLOCATION
-	help
-	  Allocate memory from heap (requires malloc()).
-endchoice
-
-config LZ4_HIGH_COMPRESSION_VARIANT
-	bool "Lz4 high compression variant"
-	help
-	  For more compression ratio at the cost of compression speed,
-	  the High Compression variant called lz4hc is available. This variant
-	  also compresses data using the lz4 block format.
-
-if LZ4_HIGH_COMPRESSION_VARIANT
-choice LZ4HC_HEAPMODE
-	prompt "How stateless HC compression functions allocate memory for their workspace"
-	default LZ4HC_HEAPMODE_HEAP
-
-config LZ4HC_HEAPMODE_STACK
-	bool "in memory stack"
-	help
-	  Allocate memory from stack (fastest).
-
-config LZ4HC_HEAPMODE_HEAP
-	bool "in memory heap"
-	depends on !LZ4_DISABLE_DYNAMIC_MEMORY_ALLOCATION
-	help
-	  Allocate memory from heap (requires malloc()).
-endchoice
-endif
-
-config LZ4_XX_HASH
-	bool "xxHash hashing algorithm"
-	help
-	  Build xxHash library included in lz4 sources.
-
-config LZ4_FRAME_SUPPORT
-	bool "LZ4 frame support"
-	select LZ4_XX_HASH
-	select LZ4_HIGH_COMPRESSION_VARIANT
-	help
-	  In order to produce compressed data compatible with lz4 command line
-	  utility, it's necessary to use the official interoperable frame format.
-	  This format is generated and decoded automatically by the lz4frame library.
-	  Its public API is described in lib/lz4frame.h.
-
-if LZ4_FRAME_SUPPORT
-choice LZ4F_HEAPMODE
-	prompt "Control how LZ4F_compressFrame() allocates the Compression State"
-	default LZ4F_HEAPMODE_STACK
-
-config LZ4F_HEAPMODE_STACK
-	bool "in memory stack"
-	help
-	  Allocate memory from stack (fastest).
-
-config LZ4F_HEAPMODE_HEAP
-	bool "in memory heap"
-	depends on !LZ4_DISABLE_DYNAMIC_MEMORY_ALLOCATION
-	help
-	  Allocate memory from heap (requires malloc()).
-endchoice
-endif
-
-endif

samples/modules/compression/compression.rst

@@ -1,3 +0,0 @@
-.. zephyr:code-sample-category:: compression
-   :name: Compression
-   :show-listing:

samples/modules/compression/lz4/CMakeLists.txt

@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-
-cmake_minimum_required(VERSION 3.20.0)
-
-find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
-project(lz4)
-
-target_sources(app PRIVATE src/main.c)

samples/modules/compression/lz4/README.rst

@@ -1,30 +0,0 @@
-.. zephyr:code-sample:: lz4
-   :name: LZ4
-
-   Compress and decompress data using the LZ4 module.
-
-Overview
-********
-
-A simple sample that can be used with any :ref:`supported board <boards>` and
-compress & decompress the user data to the console.
-
-Building and Running
-********************
-
-Add the lz4 module to your West manifest and pull it:
-
-.. code-block:: console
-
-    west config manifest.project-filter -- +lz4
-    west update
-
-The sample can be built and executed on nrf52840dk/nrf52840 as follows:
-
-.. zephyr-app-commands::
-   :zephyr-app: samples/modules/compression/lz4
-   :board: nrf52840dk/nrf52840
-   :goals: build flash
-   :compact:
-
-To build for another board, change "nrf52840dk/nrf52840" above to that board's name.

samples/modules/compression/lz4/prj.conf

@@ -1,4 +0,0 @@
-CONFIG_LZ4=y
-CONFIG_REQUIRES_FULL_LIBC=y
-CONFIG_HEAP_MEM_POOL_SIZE=16384
-CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=24576

samples/modules/compression/lz4/sample.yaml

@@ -1,24 +0,0 @@
-sample:
-  name: lz4 sample
-  description: lz4 data compression library
-common:
-  tags:
-    - compression
-    - lz4
-  min_ram: 64
-  filter: CONFIG_FULL_LIBC_SUPPORTED
-  harness: console
-  harness_config:
-    type: one_line
-    regex:
-      - "Validation done. (.*)"
-  modules:
-    - lz4
-tests:
-  sample.compression.lz4:
-    integration_platforms:
-      - mps2/an385
-      - qemu_riscv64
-    tags:
-      - compression
-      - lz4

samples/modules/compression/lz4/src/main.c

@@ -1,81 +0,0 @@
-/*
- * Copyright (c) 2020 Linumiz
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-#include <stdio.h>
-#include <zephyr/kernel.h>
-#include <zephyr/misc/lorem_ipsum.h>
-#include <string.h>
-#include <stdlib.h>
-#include "lz4.h"
-
-int main(void)
-{
-	static const char *src = LOREM_IPSUM;
-
-	const int src_size = (int)(strlen(src) + 1);
-	const int max_dst_size = LZ4_compressBound(src_size);
-
-	char *compressed_data = malloc((size_t)max_dst_size);
-
-	if (compressed_data == NULL) {
-		printk("Failed to allocate memory for compressed data\n");
-		return 0;
-	}
-
-	const int compressed_data_size = LZ4_compress_default(src,
-					compressed_data, src_size,
-					max_dst_size);
-	if (compressed_data_size <= 0) {
-		printk("Failed to compress the data\n");
-		return 0;
-	}
-
-	printk("Original Data size: %d\n", src_size);
-	printk("Compressed Data size : %d\n", compressed_data_size);
-
-	compressed_data = (char *)realloc(compressed_data,
-					  (size_t)compressed_data_size);
-	if (compressed_data == NULL) {
-		printk("Failed to re-alloc memory for compressed data\n");
-		return 0;
-	}
-
-	char * const decompressed_data = malloc(src_size);
-
-	if (decompressed_data == NULL) {
-		printk("Failed to allocate memory to decompress data\n");
-		return 0;
-	}
-
-	const int decompressed_size = LZ4_decompress_safe(compressed_data,
-				      decompressed_data, compressed_data_size,
-				      src_size);
-	if (decompressed_size < 0) {
-		printk("Failed to decompress the data\n");
-		return 0;
-	}
-
-	free(compressed_data);
-
-	if (decompressed_size >= 0) {
-		printk("Successfully decompressed some data\n");
-	}
-
-	if (decompressed_size != src_size) {
-		printk("Decompressed data is different from original\n");
-		return 0;
-	}
-
-	if (memcmp(src, decompressed_data, src_size) != 0) {
-		printk("Validation failed.\n");
-		printk("*src and *new_src are not identical\n");
-		return 0;
-	}
-
-	printk("Validation done. The string we ended up with is:\n%s\n",
-			decompressed_data);
-	return 0;
-}

submanifests/optional.yaml

@@ -16,12 +16,6 @@ manifest:
       remote: upstream
       groups:
         - optional
-    - name: lz4
-      revision: 11b8a1e22fa651b524494e55d22b69d3d9cebcfd
-      path: modules/lib/lz4
-      remote: upstream
-      groups:
-        - optional
     - name: tflite-micro
       revision: 8d404de73acf7687831e16d88e86e4f73cfddf8e
       path: optional/modules/lib/tflite-micro