text:
*New Projects should use the official [embed](https://golang.org/pkg/embed/) package instead, which was added in go 1.16.*

# binclude

[![PkgGoDev](https://pkg.go.dev/badge/github.com/lu4p/binclude)](https://pkg.go.dev/github.com/lu4p/binclude)
[![Test](https://github.com/lu4p/binclude/workflows/Test/badge.svg)](https://github.com/lu4p/binclude/actions?query=workflow%3ATest)
[![Go Report Card](https://goreportcard.com/badge/github.com/lu4p/binclude)](https://goreportcard.com/report/github.com/lu4p/binclude)
[![codecov](https://codecov.io/gh/lu4p/binclude/branch/master/graph/badge.svg)](https://codecov.io/gh/lu4p/binclude)

binclude is a tool for including static files into Go binaries.
- focuses on ease of use
- the bincluded files add no more than the filesize to the binary
- uses go/ast for typesafe parsing
- each package can have its own `binclude.FileSystem`
- `binclude.FileSystem` implements the `http.FileSystem` interface
- `ioutil` like functions `FileSystem.ReadFile`, `FileSystem.ReadDir`
- include all files/ directories under a given path by calling `binclude.Include("./path")`
- include files based on a glob pattern `binclude.IncludeGlob("./path/*.txt")`
- add file paths from a textfile `binclude.IncludeFromFile("includefile.txt")`
- high test coverage
- supports execution of executables directly from a `binclude.FileSystem` via `binexec` (os/exec wrapper)
- optional compression of files with gzip `binclude -gzip`
- debug mode to read files from disk `binclude.Debug = true`

## Install
```
GO111MODULE=on go get -u github.com/lu4p/binclude/cmd/binclude
```
## Usage
```go
package main

//go:generate binclude

import (
	"io/ioutil"
	"log"
	"path/filepath"

	"github.com/lu4p/binclude"
)

var assetPath = binclude.Include("./assets") // include ./assets with all files and subdirectories

func main() {
	binclude.Include("file.txt") // include file.txt

	// like os.Open
	f, err := BinFS.Open("file.txt")
	if err != nil {
		log.Fatalln(err)
	}

	out, err := ioutil.ReadAll(f)
	if err != nil {
		log.Fatalln(err)
	}

	log.Println(string(out))

	// like ioutil.Readfile
	content, err := BinFS.ReadFile(filepath.Join(assetPath, "asset1.txt"))
	if err != nil {
		log.Fatalln(err)
	}

	log.Println(string(content))

	// like ioutil.ReadDir
	infos, err := BinFS.ReadDir(assetPath)
	if err != nil {
		log.Fatalln(err)
	}

	for _, info := range infos {
		log.Println(info.Name())
	}
}

```
To build use:
```
go generate
go build
```

A more detailed example can be found [here](https://github.com/lu4p/binclude/tree/master/example).

## Binary size
The resulting binary, with the included files can get quite large.

You can reduce the final binary size by building without debug info (`go build -ldflags "-s -w"`) and compressing the resulting binary with [upx](https://upx.github.io/) (`upx binname`).

**Note:** If you don't need to access the compressed form of the files I would advise to just use [upx](https://upx.github.io/) and don't add seperate compression to the files.

You can add compression to the included files with `-gzip`

**Note:** decompression is optional to allow for the scenario where you want to serve compressed files for a webapp directly.

## OS / Arch Specific Includes

binclude supports including files/binaries only on specific architectures and operating systems. binclude follows the same pattern as [Go's implicit Build Constraints](https://golang.org/pkg/go/build/#hdr-Build_Constraints). It will generate files for the specific platforms like `binclude_windows.go` which contains all windows specific files.

If a file's name, matches any of the following patterns:
```
*_GOOS.go
*_GOARCH.go
*_GOOS_GOARCH.go
```
binclude will consider all files included by `binclude.Include` in this file as files which should only be included on a specific GOOS and/ or GOARCH.

For example, if you want to include a binary only on Windows you could have a file `static_windows.go` and reference the static file:
```go
package main

import "github.com/lu4p/binclude"

func bar() {
  binclude.Include("./windows-file.dll")
}
```

OS / Arch Specific Includes are used in the [binexec example](https://github.com/lu4p/binclude/tree/master/binexec/example).

**Note:** explicit build tags like `// +build debug` are not supported for including files conditionally.

## Advanced Usage
The generator can also be included into your package to allow for the code generator to run after all module dependencies are installed.
Without installing the binclude generator to the PATH seperatly.

main.go:
```go
// +build !gen

//go:generate go run -tags=gen .
package main

import (
	"github.com/lu4p/binclude"
)

func main() {
	binclude.Include("./assets")
}

```

main_gen.go:
```go
// +build gen

package main

import (
	"github.com/lu4p/binclude"
	"github.com/lu4p/binclude/bincludegen"
)

func main() {
	bincludegen.Generate(binclude.None, ".")
	// binclude.None == no compression
	// binclude.Gzip == gzip compression
}
```

To build use:
```
go generate
go build
```

## Contributing
If you find a feature missing, which you consider a useful addition please open an issue.

If you plan on implementing a new feature please open an issue so we can discuss it first (I don't want you to waste your time).

Pull requests are welcome, and will be timely reviewed and merged, please provide tests for your code, if you don't know how to test your code open a pull request and I will suggest the right testing method.


text: github: lu4p

repo_language: Go

mime_type: text/x-c


text:
on: [push, pull_request]
name: Test
jobs:
  test:
    strategy:
      matrix:
        go-version: [1.14.x,  1.15.x]
        platform: [ubuntu-latest, macos-latest, windows-latest]
    runs-on: ${{ matrix.platform }}
    steps:
    - name: Install Go
      uses: actions/setup-go@v1
      with:
        go-version: ${{ matrix.go-version }}
    - name: Check out code into the Go module directory
      uses: actions/checkout@v2
    - name: Test
      run: bash test.sh


text:
package example

//go:generate go run ../cmd/binclude

import (
	"io/ioutil"
	"log"
	"path/filepath"

	"github.com/lu4p/binclude"
)

var assetPath = binclude.Include("./assets") // include ./assets with all files and subdirectories

func main() {
	binclude.Include("file.txt") // include file.txt

	// like os.Open
	f, err := BinFS.Open("file.txt")
	if err != nil {
		log.Fatalln(err)
	}

	out, err := ioutil.ReadAll(f)
	if err != nil {
		log.Fatalln(err)
	}

	log.Println(string(out))

	// like ioutil.Readfile
	content, err := BinFS.ReadFile(filepath.Join(assetPath, "asset1.txt"))
	if err != nil {
		log.Fatalln(err)
	}

	log.Println(string(content))

	// like ioutil.ReadDir
	infos, err := BinFS.ReadDir(assetPath)
	if err != nil {
		log.Fatalln(err)
	}

	for _, info := range infos {
		log.Println(info.Name())
	}
}


text:
// Package bincludegen generates the binclude.go file
package bincludegen

import (
	"errors"
	"flag"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/lu4p/binclude"
)

var (
	operatingSystems = []string{"linux", "windows", "darwin", "freebsd", "js", "plan9", "freebsd", "dragonfly", "openbsd", "solaris", "aix", "android"}
	archs            = []string{"ppc64", "386", "amd64", "wasm", "arm", "ppc64le", "mips", "mips64", "mips64le", "mipsle", "s390x", "arm64"}

	gzip bool
)

func init() {
	flag.BoolVar(&gzip, "gzip", false, "compress files with gzip")
}

// Main1 gets called by cmd/binclude for code generation
func Main1() int {
	flag.Parse()
	compress := binclude.None
	if gzip {
		compress = binclude.Gzip
	}

	log.SetPrefix("[binclude] ")

	err := Generate(compress, ".")
	if err != nil {
		log.Println("failed:", err)
		return 1
	}

	return 0
}

// Generate a binclude.go file for the current working directory
func Generate(compress binclude.Compression, dir string) error {
	fset := token.NewFileSet()
	filter := func(info os.FileInfo) bool {
		if strings.Contains(info.Name(), "_test") {
			return false
		}
		return !strings.HasPrefix(info.Name(), "binclude")
	}

	pkgs, err := parser.ParseDir(fset, dir, filter, parser.ParseComments)
	if err != nil {
		return err
	}

	if len(pkgs) > 1 {
		return errors.New("more than one package in the current directory")
	}

	var (
		pkg     *ast.Package
		pkgName string
	)

	for pkgName, pkg = range pkgs {
		break // only get the first package
	}

	includedFiles, err := detectIncluded(pkg)
	if err != nil {
		return err
	}

	fileSystems, err := buildFS(includedFiles)
	if err != nil {
		return err
	}

	for _, fs := range fileSystems {
		if err := fs.Compress(compress); err != nil {
			return err
		}
	}

	return generateFiles(dir, pkgName, fileSystems)
}

func buildFS(includedFiles []includedFile) (map[string]*binclude.FileSystem, error) {
	const bincludeName = "binclude"
	fileSystems := make(map[string]*binclude.FileSystem)
	var buildTag string

	fileSystems["default"] = &binclude.FileSystem{}
	fileSystems["default"].Files = make(binclude.Files)

	var walkFn filepath.WalkFunc = func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		var content []byte

		if !info.IsDir() {
			content, err = ioutil.ReadFile(path)
			if err != nil {
				return err
			}

		}

		path = filepath.ToSlash(path)

		if fileSystems[buildTag] == nil {
			fileSystems[buildTag] = &binclude.FileSystem{}
			fileSystems[buildTag].Files = make(binclude.Files)
		}
		createFile(fileSystems[buildTag], path, &binclude.File{
			Filename: info.Name(),
			Mode:     info.Mode(),
			ModTime:  info.ModTime(),
			Content:  content,
		})

		return nil
	}

	for _, file := range includedFiles {
		buildTag = ""

		for _, arch := range archs {
			if strings.HasSuffix(file.goFile, arch+".go") {
				buildTag = "_" + arch
			}
		}

		for _, sys := range operatingSystems {
			if strings.HasSuffix(file.goFile, sys+buildTag+".go") {
				buildTag = "_" + sys + buildTag
			}
		}

		if len(buildTag) == 0 {
			buildTag = "default"
		}

		err := filepath.Walk(file.includedPath, walkFn)
		if err != nil {
			return nil, err
		}
	}

	return fileSystems, nil
}

type includedFile struct {
	includedPath, goFile string
}

func detectIncluded(pkg *ast.Package) ([]includedFile, error) {
	var includedFiles []includedFile

	var currentGoFile string

	visit := func(node ast.Node) bool {
		if node == nil {
			return true
		}

		call, ok := node.(*ast.CallExpr)
		if !ok {
			return true
		}
		sel, ok := call.Fun.(*ast.SelectorExpr)
		if !ok {
			return true
		}

		v, ok := sel.X.(*ast.Ident)
		if !ok {
			return true
		}

		if v.Name != "binclude" {
			return true
		}

		lit, ok := call.Args[0].(*ast.BasicLit)
		if !ok || lit.Kind != token.STRING {
			log.Fatalln("argument is not string literal")
		}

		value, err := strconv.Unquote(lit.Value)
		if err != nil {
			log.Fatalln("cannot unquote string:", err)
		}

		switch sel.Sel.Name {
		case "Include":
			includedFiles = append(includedFiles, includedFile{
				goFile:       currentGoFile,
				includedPath: value,
			})
		case "IncludeFromFile":
			includedFiles = includeFromFile(value, currentGoFile, includedFiles)
		case "IncludeGlob":
			includedFiles = includeGlob(value, currentGoFile, includedFiles)
		}

		return true
	}

	for path, file := range pkg.Files {
		currentGoFile = path
		ast.Inspect(file, visit)
	}

	for i, file := range includedFiles {
		var err error

		if filepath.IsAbs(file.includedPath) {
			return nil, errors.New("only supports relative include paths")
		}

		_, err = os.Stat(file.includedPath)
		if err != nil {
			return nil, err
		}

		includedFiles[i].includedPath = strings.TrimPrefix(file.includedPath, "./")
	}

	return includedFiles, nil
}

func includeFromFile(value, currentGoFile string, includedFiles []includedFile) []includedFile {
	content, err := ioutil.ReadFile(value)
	if err != nil {
		log.Fatalln("cannot read includefile:", value, "err:", err)
	}

	paths := strings.Split(string(content), "\n")
	for i := 0; i < len(paths); i++ {
		paths[i] = strings.TrimSpace(paths[i])
		if paths[i] == "" {
			paths = remove(paths, i)
			i-- // reset position by one because an element was removed
		}
	}

	for _, path := range paths {
		includedFiles = append(includedFiles, includedFile{
			goFile:       currentGoFile,
			includedPath: path,
		})
	}

	return includedFiles
}

func includeGlob(pattern, currentGoFile string, includedFiles []includedFile) []includedFile {
	matches, err := filepath.Glob(pattern)
	if err != nil {
		log.Fatalln("cannot glob:", pattern, "err:", err)
	}

	for _, match := range matches {
		includedFiles = append(includedFiles, includedFile{
			goFile:       currentGoFile,
			includedPath: match,
		})
	}
	return includedFiles
}

func remove(slice []string, s int) []string {
	return append(slice[:s], slice[s+1:]...)
}

func createFile(fs *binclude.FileSystem, path string, file *binclude.File) error {
	dir := filepath.Dir(path)
	_, err := fs.Stat(dir)
	if err != nil {
		mkdir(fs, dir, os.ModePerm)
	}

	path = strings.TrimPrefix(path, "./")
	fs.Files[path] = file
	return nil
}

func mkdir(fs *binclude.FileSystem, name string, perm os.FileMode) {
	name = strings.TrimPrefix(name, "./")

	fs.Files[name] = &binclude.File{
		Filename: filepath.Base(name),
		Mode:     os.ModeDir | perm,
		ModTime:  time.Now(),
		Content:  nil,
	}
}


text:
package bincludegen_test

import (
	"flag"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/lu4p/binclude/bincludegen"
	"github.com/rogpeppe/go-internal/gotooltest"
	"github.com/rogpeppe/go-internal/testscript"
)

func TestMain(m *testing.M) {
	os.Exit(testscript.RunMain(m, map[string]func() int{
		"binclude": bincludegen.Main1,
	}))
}

var update = flag.Bool("u", false, "update testscript output files")

func TestScripts(t *testing.T) {
	t.Parallel()

	wd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}

	data := `module test/main

replace github.com/lu4p/binclude => ` + filepath.Join(wd, "..") + `

require (
	github.com/lu4p/binclude v1.0.0
)`

	modPath := filepath.Join(wd, "go.mod.txt")
	err = ioutil.WriteFile(modPath, []byte(data), 0644)
	if err != nil {
		t.Fatal(err)
	}

	p := testscript.Params{
		Dir: filepath.Join("testdata", "scripts"),
		Setup: func(env *testscript.Env) error {
			os.Getwd()
			env.Vars = append(env.Vars,
				"MOD_PATH="+modPath,
			)
			return nil
		},
		Cmds: map[string]func(ts *testscript.TestScript, neg bool, args []string){
			"bincmp": bincmp,
		},
		UpdateScripts: *update,
	}
	if err := gotooltest.Setup(&p); err != nil {
		t.Fatal(err)
	}
	testscript.Run(t, p)
}

func bincmp(ts *testscript.TestScript, neg bool, args []string) {
	if len(args) != 2 {
		ts.Fatalf("usage: bincmp file1 file2")
	}
	data1 := ts.ReadFile(args[0])
	data2 := ts.ReadFile(args[1])
	if neg {
		if data1 == data2 {
			ts.Fatalf("%s and %s don't differ",
				args[0], args[1])
		}
		return
	}
	if data1 != data2 {
		sizeDiff := len(data2) - len(data1)
		ts.Fatalf("%s and %s differ; size diff: %+d",
			args[0], args[1], sizeDiff)
	}
}


text:
package bincludegen

import (
	"bytes"
	"fmt"
	"go/format"
	"io/ioutil"
	"path/filepath"
	"sort"

	"github.com/lu4p/binclude"
)

func generateCode(pkgName string, fs *binclude.FileSystem, buildTag string) *bytes.Buffer {
	b := bytes.NewBuffer(nil)

	header := `// Code generated by https://github.com/lu4p/binclude; DO NOT EDIT.

package ` + pkgName + "\n"

	b.WriteString(header)

	b.WriteString("import (\n")
	b.WriteString("\"github.com/lu4p/binclude\"\n")
	if len(fs.Files) > 0 {
		b.WriteString("\"time\"\n")
	}
	b.WriteString(")\n")

	fsName := "BinFS"
	if buildTag != "default" {
		fsName = "_binfs" + buildTag
	}

	fsCode(fs, b, fsName)

	return b
}

func generateFiles(dir, pkgName string, fileSystems map[string]*binclude.FileSystem) error {
	for buildTag, fs := range fileSystems {
		code := generateFile(pkgName, fs, buildTag)

		path := filepath.Join(dir, "binclude"+buildTag+".go")
		if buildTag == "default" {
			path = filepath.Join(dir, "binclude.go")
		}

		if err := writeCodeToFile(path, code); err != nil {
			return err
		}
	}
	return nil
}

func generateFile(pkgName string, fs *binclude.FileSystem, buildTag string) []byte {
	b := generateCode(pkgName, fs, buildTag)

	if buildTag == "default" {
		return b.Bytes()
	}

	initFunc := `
func init() {
	BinFS.Lock()
	for path, file := range _binfs` + buildTag + `.Files {
		BinFS.Files[path] = file
	}
	BinFS.Unlock()
}`

	b.WriteString(initFunc)
	return b.Bytes()
}

func writeCodeToFile(filename string, code []byte) error {
	fmtCode, err := format.Source([]byte(code))
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filename, fmtCode, 0o666)
}

func fsCode(fs *binclude.FileSystem, b *bytes.Buffer, fsName string) {
	fmt.Fprintf(b, "var %s = &binclude.FileSystem{Files: binclude.Files{\n", fsName)

	var paths []string
	for path := range fs.Files {
		paths = append(paths, path)
	}

	sort.Strings(paths)

	for _, path := range paths {
		file := fs.Files[path]
		fileCode(file, b, path)
	}

	b.WriteString("}}\n")
}

func fileCode(f *binclude.File, b *bytes.Buffer, path string) {
	fmt.Fprintf(b, "%q:{\n", path)

	fmt.Fprintf(b, `Filename: %q, Mode: %O, ModTime: time.Unix(%d,%d), Compression: %d,`,
		f.Filename, f.Mode, f.ModTime.Unix(), f.ModTime.UnixNano(), f.Compression)

	if f.Content != nil {
		fmt.Fprintf(b, "\nContent: []byte(%q),", f.Content)
	}

	b.WriteString("\n},\n")
}


text:
package bincludegen_test

import (
	"testing"

	"github.com/lu4p/binclude"
	"github.com/lu4p/binclude/bincludegen"
)

func BenchmarkGenerate(b *testing.B) {
	err := bincludegen.Generate(binclude.None, "./testdata/bench")
	if err != nil {
		b.Fatal(err)
	}
}


text:
binclude
cp $MOD_PATH go.mod
go build
exec ./main$exe
cmp stderr main.stderr

-- main.go --
package main

import (
	"io/ioutil"

	"github.com/lu4p/binclude"
)

func main() {
	binclude.Include("./assets")
	binclude.Include("file.txt")

	f, err := BinFS.Open("file.txt")
	if err != nil {
		panic(err)
	}

	out, err := ioutil.ReadAll(f)
	if err != nil {
		panic(err)
	}

	println(string(out))

	infos, err := BinFS.ReadDir("./assets")
	if err != nil {
		panic(err)
	}

	for _, info := range infos {
		println(info.Name())
	}
}

-- binclude.go --
// Code generated by binclude; DO NOT EDIT.

package main

import (
	"github.com/lu4p/binclude"
	"time"
)

var (
	_binclude0	= []byte("asset1")
	_binclude1	= []byte("subdirasset1")
	_binclude2	= []byte("subdirasset1")
	_binclude3	= []byte("file.txt")
	_binclude4	= []byte("asset1")
)
var BinFS = binclude.FileSystem{"assets/asset2.txt": {Filename: "asset2.txt", Mode: 420, ModTime: time.Unix(1592103958, 0), Content: _binclude0}, "assets/subdir": {Filename: "subdir", Mode: 2147484141, ModTime: time.Unix(1592104033, 0), Content: nil}, "assets/subdir/subdirasset1.txt": {Filename: "subdirasset1.txt", Mode: 420, ModTime: time.Unix(1592104027, 0), Content: _binclude1}, "assets/subdir/subdirasset2.txt": {Filename: "subdirasset2.txt", Mode: 420, ModTime: time.Unix(1592104033, 0), Content: _binclude2}, "file.txt": {Filename: "file.txt", Mode: 420, ModTime: time.Unix(1592104011, 0), Content: _binclude3}, "assets": {Filename: "assets", Mode: 2147484141, ModTime: time.Unix(1592518785, 0), Content: nil}, "assets/asset1.txt": {Filename: "asset1.txt", Mode: 420, ModTime: time.Unix(1592103949, 0), Content: _binclude4}}

-- assets/asset1.txt --
asset1

-- assets/asset2.txt --
asset2

-- subdir/subdirasset1.txt --
subdirasset1

-- subdir/subdirasset2.txt --
subdirasset2

-- file.txt --
file
-- main.stderr --
file

asset1.txt
asset2.txt


text:
[!windows] chmod 000 main.go
[!windows] ! binclude

-- main.go --
package main

import (
	"github.com/lu4p/binclude"
)

func main() {
	binclude.Include("./assets")

	infos, err := BinFS.ReadDir("./assets")
	if err != nil {
		panic(err)
	}

	for _, info := range infos {
		println(info.Name())
	}
}

-- assets/asset1.txt --
asset1

-- assets/asset2.txt --
asset2


text:
binclude
cp $MOD_PATH go.mod
go build
exec ./main$exe
cmp stderr main.stderr

-- main.go --
package main

import (
	"io/ioutil"

	"github.com/lu4p/binclude"
)

func main() {
	binclude.IncludeFromFile("includefile.txt")

	f, err := BinFS.Open("file.txt")
	if err != nil {
		panic(err)
	}

	out, err := ioutil.ReadAll(f)
	if err != nil {
		panic(err)
	}

	println(string(out))

	infos, err := BinFS.ReadDir("./assets")
	if err != nil {
		panic(err)
	}

	for _, info := range infos {
		println(info.Name())
	}
}

-- main_test.go --
package main_test
// testpackages should be excluded

-- includefile.txt --
./assets
file.txt

-- assets/asset1.txt --
asset1

-- assets/asset2.txt --
asset2

-- subdir/subdirasset1.txt --
subdirasset1

-- subdir/subdirasset2.txt --
subdirasset2

-- file.txt --
file

-- main.stderr --
file

asset1.txt
asset2.txt


text:
binclude
cp $MOD_PATH go.mod
go run .
[linux] cmp stdout main.linux.stdout
[windows] cmp stdout main.windows.stdout
[darwin] cmp stdout main.darwin.stdout

-- main.go --
package main

func main() {
	printosfile()
}

-- second_darwin.go --
package main

import (
	"fmt"
	"log"

	"github.com/lu4p/binclude"
)

func printosfile() {
	binclude.Include("darwin.txt")
	content, err := BinFS.ReadFile("darwin.txt")
	if err != nil {
		log.Fatalln(err)
	}

	fmt.Println(string(content))
}

-- second_linux.go --
package main

import (
	"fmt"
	"log"

	"github.com/lu4p/binclude"
)

func printosfile() {
	binclude.Include("linux.txt")
	content, err := BinFS.ReadFile("linux.txt")
	if err != nil {
		log.Fatalln(err)
	}

	fmt.Println(string(content))
}

-- second_windows.go --
package main

import (
	"fmt"
	"log"

	"github.com/lu4p/binclude"
)

func printosfile() {
	binclude.Include("windows.txt")
	content, err := BinFS.ReadFile("windows.txt")
	if err != nil {
		log.Fatalln(err)
	}

	fmt.Println(string(content))
}

-- linux.txt --
linux
-- darwin.txt --
darwin
-- windows.txt --
windows
-- main.linux.stdout --
linux

-- main.darwin.stdout --
darwin

-- main.windows.stdout --
windows

-- main.stdout --


text:
[!windows] chmod 544 .
[!windows] ! binclude

-- main.go --
package main

import (
	"io/ioutil"
	"fmt"

	"github.com/lu4p/binclude"
)

func main() {
	binclude.Include("./assets")
	binclude.Include("file.txt")

	f, err := BinFS.Open("file.txt")
	if err != nil {
		panic(err)
	}

	out, err := ioutil.ReadAll(f)
	if err != nil {
		panic(err)
	}

	fmt.Println(string(out))

	infos, err := BinFS.ReadDir("./assets")
	if err != nil {
		panic(err)
	}

	for _, info := range infos {
		fmt.Println(info.Name())
	}
}

-- assets/asset1.txt --
asset1

-- assets/asset2.txt --
asset2

-- subdir/subdirasset1.txt --
subdirasset1

-- subdir/subdirasset2.txt --
subdirasset2

-- file.txt --
file

-- main.stdout --
file

asset1.txt
asset2.txt


text:
! exec binclude

-- main.go --
package main

import (
	"github.com/lu4p/binclude"
)

func main() {
	binclude.Include("./assets")

	anotherfile()

	infos, err := BinFS.ReadDir("./assets")
	if err != nil {
		panic(err)
	}

	for _, info := range infos {
		println("Name:", info.Name())
	}
}

-- anotherfile.go --
package wrongpackage

import (
	"io/ioutil"

	"github.com/lu4p/binclude"
)

func anotherfile() {
	binclude.Include("file.txt")

	f, err := BinFS.Open("file.txt")
	if err != nil {
		panic(err)
	}

	out, err := ioutil.ReadAll(f)
	if err != nil {
		panic(err)
	}

	println(string(out))
}

-- assets/asset1.txt --
asset1

-- assets/asset2.txt --
asset2

-- subdir/subdirasset1.txt --
subdirasset1

-- subdir/subdirasset2.txt --
subdirasset2

-- file.txt --
file
-- main.stderr --
file

Name: asset1.txt
Name: asset2.txt


text:
[!windows] chmod 000 assets
[!windows] ! binclude
[!windows] chmod 755 assets
[!windows] chmod 000 assets/asset1.txt
[!windows] ! binclude

-- main.go --
package main

import (
	"github.com/lu4p/binclude"
)

func main() {
	binclude.Include("./assets")

	infos, err := BinFS.ReadDir("./assets")
	if err != nil {
		panic(err)
	}

	for _, info := range infos {
		println(info.Name())
	}
}

-- assets/asset1.txt --
asset1

-- assets/asset2.txt --
asset2


text:
binclude
cp $MOD_PATH go.mod
go build
exec ./main$exe
cmp stderr main.stderr

-- main.go --
package main

import (
	"io/ioutil"

	"github.com/lu4p/binclude"
)

func main() {
	binclude.Debug = true

	infos, err := ioutil.ReadDir("assets")
	if err != nil {
		panic(err)
	}

	for _, info := range infos {
		println(info.Name())
	}

	infos, err = BinFS.ReadDir("./assets")
	if err != nil {
		panic(err)
	}

	for _, info := range infos {
		println(info.Name())
	}
}

-- assets/asset1.txt --
asset1

-- assets/asset2.txt --
asset2

-- subdir/subdirasset1.txt --
subdirasset1

-- subdir/subdirasset2.txt --
subdirasset2

-- main.stderr --
asset1.txt
asset2.txt
asset1.txt
asset2.txt


text:
! binclude

-- assets/asset1.txt --
asset1

-- assets/asset2.txt --
asset2

-- subdir/subdirasset1.txt --
subdirasset1

-- subdir/subdirasset2.txt --
subdirasset2


text:
binclude
cp $MOD_PATH go.mod
go build
exec ./main$exe
cmp stderr main.stderr

-- main.go --
package main

import (
	"io/ioutil"

	"github.com/lu4p/binclude"
)

func main() {
	binclude.Include("./assets")
	binclude.Include("file.txt")

	f, err := BinFS.Open("file.txt")
	if err != nil {
		panic(err)
	}

	out, err := ioutil.ReadAll(f)
	if err != nil {
		panic(err)
	}

	println(string(out))

	infos, err := BinFS.ReadDir("./assets")
	if err != nil {
		panic(err)
	}

	for _, info := range infos {
		println(info.Name())
	}
}

-- assets/asset1.txt --
asset1

-- assets/asset2.txt --
asset2

-- subdir/subdirasset1.txt --
subdirasset1

-- subdir/subdirasset2.txt --
subdirasset2

-- file.txt --
file

-- main.stderr --
file

asset1.txt
asset2.txt


text:
! binclude

-- main.go --
package main

import (
	"github.com/lu4p/binclude"
)

func main() {
	binclude.Include("./assets")
}


text:
! binclude
cp $MOD_PATH go.mod

! go build

-- main.go --
package main

import (
	"fmt"

	"github.com/lu4p/binclude"
)

func main() {
	binclude.Include("/absolute")

	infos, err := BinFS.ReadDir("/absolute")
	if err != nil {
		panic(err)
	}

	for _, info := range infos {
		fmt.Println(info.Name())
	}
}


text:
binclude -gzip
cp $MOD_PATH go.mod
go build
exec ./main$exe
cmp stdout main.stdout

-- main.go --
package main

import (
	"io/ioutil"
	"fmt"

	"github.com/lu4p/binclude"
)

func main() {
	binclude.Include("./assets")
	binclude.Include("file.txt")

	err := BinFS.Decompress()
	if err != nil{
		panic(err)
	}

	f, err := BinFS.Open("file.txt")
	if err != nil {
		panic(err)
	}

	out, err := ioutil.ReadAll(f)
	if err != nil {
		panic(err)
	}

	fmt.Println(string(out))

	infos, err := BinFS.ReadDir("./assets")
	if err != nil {
		panic(err)
	}

	for _, info := range infos {
		fmt.Println(info.Name())
	}
}

-- assets/asset1.txt --
asset1

-- assets/asset2.txt --
asset2

-- subdir/subdirasset1.txt --
subdirasset1

-- subdir/subdirasset2.txt --
subdirasset2

-- file.txt --
file

-- main.stdout --
file

asset1.txt
asset2.txt


text:
binclude
cp $MOD_PATH go.mod
go build
exec ./main$exe
cmp stderr main.stderr

-- main.go --
package main

import (
	"github.com/lu4p/binclude"
)

func main() {
	binclude.Include("./assets")

	anotherfile()

	infos, err := BinFS.ReadDir("./assets")
	if err != nil {
		panic(err)
	}

	for _, info := range infos {
		println("Name:", info.Name())
	}
}

-- anotherfile.go --
package main

import (
	"io/ioutil"

	"github.com/lu4p/binclude"
)

func anotherfile() {
	binclude.Include("file.txt")

	f, err := BinFS.Open("file.txt")
	if err != nil {
		panic(err)
	}

	out, err := ioutil.ReadAll(f)
	if err != nil {
		panic(err)
	}

	println(string(out))
}

-- assets/asset1.txt --
asset1

-- assets/asset2.txt --
asset2

-- subdir/subdirasset1.txt --
subdirasset1

-- subdir/subdirasset2.txt --
subdirasset2

-- file.txt --
file
-- main.stderr --
file

Name: asset1.txt
Name: asset2.txt


text:
binclude
cp $MOD_PATH go.mod
go build
exec ./main$exe
cmp stderr main.stderr

-- main.go --
package main

import (
	"github.com/lu4p/binclude"
)

func main() {
	binclude.IncludeGlob("./assets/*.txt")

	infos, err := BinFS.ReadDir("./assets")
	if err != nil {
		panic(err)
	}

	for _, info := range infos {
		println(info.Name())
	}
}

-- assets/asset1.txt --
asset1

-- assets/asset2.txt --
asset2

-- subdir/subdirasset1.txt --
subdirasset1

-- subdir/subdirasset2.txt --
subdirasset2

-- main.stderr --
asset1.txt
asset2.txt


text:
package main

import "github.com/lu4p/binclude"

//go:generate go build -o=./includedPrg/includedPrg ./includedPrg
func main() {
	binclude.Include("./testdata/bench/includedPrg/includedPrg")
}


text:
package main

import "fmt"

func main() {
	fmt.Println("Hello world!")
}


text:
# Golang CircleCI 2.0 configuration file
#
# Check https://circleci.com/docs/2.0/language-go/ for more details
version: 2
jobs:
  build:
    docker:
      # specify the version
      - image: circleci/golang:1.15

    steps:
      # specify any bash command here prefixed with `run: `
      - run: git clone https://github.com/lu4p/binclude.git .
      - run: bash test.sh
      - run: bash <(curl -s https://codecov.io/bash)


text:
package binexec_test

import (
	"context"
	"io/ioutil"
	"log"
	"testing"

	"github.com/lu4p/binclude/binexec"
	"github.com/lu4p/binclude/binexec/example"
)

var BinFS = example.BinFS

var testprg = example.Testprg

func init() {
	for path := range BinFS.Files {
		log.Println(path)
	}
}

func TestRun(t *testing.T) {
	cmd, err := binexec.Command(BinFS, testprg)
	if err != nil {
		t.Fatal("cannot initialize cmd", err)
	}

	err = cmd.Run()
	if err != nil {
		t.Fatal("cannot execute cmd", err)
	}
}

func TestNonexistent(t *testing.T) {
	_, err := binexec.Command(BinFS, "Nonexistent")
	if err == nil {
		t.Fatal("can initialize cmd on Nonexistent file")
	}

	_, err = binexec.CommandContext(context.Background(), BinFS, "Nonexistent")
	if err == nil {
		t.Fatal("can initialize cmd on Nonexistent file")
	}

}

func TestStart(t *testing.T) {
	cmd, err := binexec.Command(BinFS, testprg)
	if err != nil {
		t.Fatal("cannot initialize cmd", err)
	}
	t.Log(cmd)

	_, err = cmd.StdinPipe()
	if err != nil {
		t.Fatal(err)
	}

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		t.Fatal(err)
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		t.Fatal(err)
	}

	err = cmd.Start()
	if err != nil {
		t.Fatal("cannot execute cmd", err)
	}

	out, err := ioutil.ReadAll(stderr)
	if err != nil {
		t.Fatal(err)
	}

	if _, err := ioutil.ReadAll(stdout); err != nil {
		t.Fatal(err)
	}

	err = cmd.Wait()
	if err != nil {
		t.Fatal("wait:", err)
	}

	if string(out) != "Hello world!\n" {
		t.Fatal("unexpected output:", string(out))
	}
}

func TestRunContext(t *testing.T) {
	cmd, err := binexec.CommandContext(context.Background(), BinFS, testprg)
	if err != nil {
		t.Fatal("cannot initialize cmd", err)
	}

	err = cmd.Run()
	if err != nil {
		t.Fatal("cannot execute cmd", err)
	}
}


text:
// Package binexec implements a wrapper for os/exec
package binexec

import (
	"context"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/lu4p/binclude"
)

// Cmd same as Cmd in the os/exec package
type Cmd struct {
	OsCmd *exec.Cmd
	// Cache if set to true the binary won't be deleted after execution.
	// If the ModTime of the cached binclude file changes the cache gets invalidated automtically.
	Cache bool
}

// Command similar to Command in the os/exec package,
// but copies the executeable to run from bincludePath
// to the host os.
func Command(fs *binclude.FileSystem, bincludePath string, arg ...string) (*Cmd, error) {
	execPath, err := copyCommand(fs, bincludePath)
	if err != nil {
		return nil, err
	}

	cmd := Cmd{
		OsCmd: exec.Command(execPath, arg...),
	}

	return &cmd, nil
}

// copyCommand copy a file from binclude.FileSystem to os.UserCacheDir()
func copyCommand(fs *binclude.FileSystem, bincludePath string) (string, error) {
	dir, _ := os.UserCacheDir()

	info, err := fs.Stat(bincludePath)
	if err != nil {
		return "", err
	}

	nanoSec := strconv.Itoa(info.ModTime().Nanosecond())

	namePart := "_" + filepath.Base(bincludePath)

	execPath := filepath.Join(dir, nanoSec+namePart)

	// exit early if file is already cached
	_, err = os.Stat(execPath)
	if err == nil {
		return execPath, nil
	}

	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", err
	}

	// remove invalidated cache files
	for _, info := range infos {
		if strings.HasSuffix(info.Name(), namePart) {
			os.Remove(filepath.Join(dir, info.Name())) // don't check for error because we don't really care if the file is removed
			// no break because there could be multiple cached versions
		}
	}

	return execPath, fs.CopyFile(bincludePath, execPath)
}

// CommandContext similar to CommandContext in the os/exec
// package but copies the executeable to run from bincludePath
// to the host os.
func CommandContext(ctx context.Context, fs *binclude.FileSystem, bincludePath string, arg ...string) (*Cmd, error) {
	execPath, err := copyCommand(fs, bincludePath)
	if err != nil {
		return nil, err
	}

	cmd := Cmd{
		OsCmd: exec.CommandContext(ctx, execPath, arg...),
	}

	return &cmd, nil
}

// Run is similar to (*Cmd).Run() in the os/exec package,
// but deletes the executable at Cmd.Path if c.Cache is false
func (c *Cmd) Run() error {
	if !c.Cache {
		defer os.Remove(c.OsCmd.Path)
	}

	return c.OsCmd.Run()
}

// Start same as (*Cmd).Start() in the os/exec package
func (c *Cmd) Start() error {
	return c.OsCmd.Start()
}

// StderrPipe same as (*Cmd).StderrPipe() in the os/exec package
func (c *Cmd) StderrPipe() (io.ReadCloser, error) {
	return c.OsCmd.StderrPipe()
}

// StdinPipe same as (*Cmd).StdinPipe() in the os/exec package
func (c *Cmd) StdinPipe() (io.WriteCloser, error) {
	return c.OsCmd.StdinPipe()
}

// StdoutPipe same as (*Cmd).StdoutPipe() in the os/exec package
func (c *Cmd) StdoutPipe() (io.ReadCloser, error) {
	return c.OsCmd.StdoutPipe()
}

// String same as (*Cmd).String() in the os/exec package
func (c *Cmd) String() string {
	return c.OsCmd.String()
}

// Wait is similar to (*Cmd).Wait() in the os/exec package,
// but deletes the executable at Cmd.Path if c.Cache is false
func (c *Cmd) Wait() error {
	if !c.Cache {
		defer os.Remove(c.OsCmd.Path)
	}

	return c.OsCmd.Wait()
}


text:
package example

import (
	"log"

	"github.com/lu4p/binclude/binexec"
)

//go:generate env GOOS=windows go build -o ./testprg/testprg.exe ./testprg
//go:generate env GOOS=linux go build -o ./testprg/testprg ./testprg
//go:generate env GOOS=darwin go build -o ./testprg/testprg_darwin ./testprg

//go:generate go run ../../cmd/binclude/

// Example shows the example usage for binexec
func Example() {
	cmd, err := binexec.Command(BinFS, Testprg)
	if err != nil {
		log.Fatal(err)
	}

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}


text:
package example

import (
	"github.com/lu4p/binclude"
)

// Testprg the program to run for the Example
var Testprg = "./testprg/testprg_darwin"

func darwin() {
	binclude.Include("./testprg/testprg_darwin")
}


text:
package example

import (
	"github.com/lu4p/binclude"
)

// Testprg the program to run for the Example
var Testprg = "./testprg/testprg"

func linux() {
	binclude.Include("./testprg/testprg")
}


text:
package example

import (
	"github.com/lu4p/binclude"
)

// Testprg the program to run for the Example
var Testprg = "./testprg/testprg.exe"

func windows() {
	binclude.Include("./testprg/testprg.exe")
}


text:
// Code generated by https://github.com/lu4p/binclude; DO NOT EDIT.

package example

import (
	"github.com/lu4p/binclude"
)

var BinFS = &binclude.FileSystem{Files: binclude.Files{}}


text:
package main

func main() {
	println("Hello world!")
}


text:
package main

import (
	"os"

	"github.com/lu4p/binclude/bincludegen"
)

func main() {
	os.Exit(bincludegen.Main1())
}


text:
# Arduino_UKF_Library
This is a compact Unscented Kalman Filter (UKF) library for Teensy4.0/Arduino system (or real time embedded system in general).
- It's not using Eigen (small source code - more simple to understand).
- It's not using C++ Standard Library/std (for embedded consideration).
- If you set `SYSTEM_IMPLEMENTATION` to `SYSTEM_IMPLEMENTATION_EMBEDDED_NO_PRINT` in `konfig.h`, the code is platform agnostic (not using any library beside these C header files: `stdlib.h`, `stdint.h`, `stdbool.h`, `string.h`, and `math.h`).
- There's no malloc/new/free dynamic memory allocation for real time application (but using heavy stack local variables, so you need to run it through static memory analyzer if you are really concerned about implement this in mission critical hard real time application).

# The Background
The Unscented Kalman Filter is designed to handle state variable estimation where the system is highly nonlinear that the Extended Kalman Flter (EKF) will do the job poorly, or even fail to do so (e.g. for non-differentiable nonlinear system). Unfortunately, the algorithm is quite complex compared to EKF. With that in mind, I made this library for any student who want to learn the structure of UKF, the computer code implementation of it, and how to use the filter for a nontrivial problem.

This library is made with specific goal for educational purpose (I've made decision to sacrifice speed to get best code readability I could get) while still capable of tackling real-time control system implementation (the code is computed under **250 us** for non trivia application! See *[Some Benchmark](README.md#some-benchmark)* section below). I strongly suggest you to [learn the EKF first in my other repository (Arduino_EKF_Library)](https://github.com/pronenewbits/Arduino_EKF_Library) before delve deeper into UKF, because many UKF motivation and design lies on analyzing EKF weakness (specifically, on its need for linearization).

Without further ado, first some definition:
![UKF Definition](UKF_Definition.png "Click to maximize if the image rescaling make you feel dizzy")

Many authors blended the unscented transformation into kalman filter structure (even in the Jeffrey Uhlmann's paper) to make it into a familiar KF structure. But in this implementation, I prefer to emphasize the unscented transformation structure in the hope you can get the big idea of the transformation more easily. That make the implementation somewhat different from a typical UKF formulation, but in essence it's all the same.

The UKF algorithm can be descibed as:
![UKF Calculation](UKF_Calculation.png "Click to maximize if the image rescaling make you feel dizzy")

# How to Use
The UKF code is self contained and can be accessed in the folder [ukf_engl](ukf_engl) (this is the template project). Inside you will find these files:
- `matrix.h/cpp` : The backbone of all my code in this account. This files contain the class for Matrix operation.
- `ukf.h/cpp` : The source files of the Unscented Kalman Filter Class.
- `konfig.h` : The configuration file.
- `ukf_engl.ino` : The arduino main file (this is only the template file).

For custom implementation, typically you only need to modify `konfig.h` and `*.ino` files. Where basically you need to:
1. Set the length of `X, U, Z` vectors and sampling time `dt` in `konfig.h`, depend on your model.
2. Implement the nonlinear update function `f(x)`, measurement function `h(x)`, initialization value `P(k=0)`, and `Rv & Rn` constants value in the `*.ino` file.

After that, you only need to initialize the UKF class, set the non-zero initialization matrix by calling `UKF::vReset(X_INIT, P_INIT, Rv_INIT, Rn_INIT)` function at initialization, and call `UKF::bUpdate(Y,U)` function every sampling time.

To see how you can implement the library in non-trivial application, I've implemented 2 example:
1.  [ukf_example1_pendulum](ukf_example1_pendulum). This example simulate the damped pendulum. See the [README file](ukf_example1_pendulum/README.md) inside the folder to get more information.
2.  [ukf_example2_imu](ukf_example2_imu). This example process IMU (Inertial Measurement Unit) data using sensor MPU9250. See the [README file](ukf_example2_imu/README.md) inside the folder to get more information.

*For Arduino configuration (`SYSTEM_IMPLEMENTATION` is set to `SYSTEM_IMPLEMENTATION_EMBEDDED_ARDUINO` in `konfig.h`):
The code is tested on compiler Arduino IDE 1.8.10 and hardware Teensy 4.0 Platform.

*For PC configuration (`SYSTEM_IMPLEMENTATION` is set to `SYSTEM_IMPLEMENTATION_PC` in `konfig.h`):
The code is tested on compiler Qt Creator 4.8.2 and typical PC Platform.

**Important note: For Teensy 4.0, I encounter RAM limitation where the `MATRIX_MAXIMUM_SIZE` can't be more than 28 (if you are using double precision) or 40 (if using single precision). If you already set more than that, your Teensy might be unable to be programmed (stack overflow make the bootloader program goes awry?). The solution is simply to change the `MATRIX_MAXIMUM_SIZE` to be less than that, compile & upload the code from the compiler. The IDE then will protest that it cannot find the Teensy board. DON'T PANIC. Click the program button on the Teensy board to force the bootloader to restart and download the firmware from the computer.**

next step: implement [memory pool](https://en.wikipedia.org/wiki/Memory_pool) for Matrix library!

# Some Benchmark
The computation time needed to compute one iteration of `UKF::bUpdate(Y,U)` function are:
1. [ukf_example1_pendulum](ukf_example1_pendulum) (2 state, no input, 2 output): **45 us** to compute one iteration (single precision math) or **53 us** (double precision). The result, plotted using [Scilab](https://www.scilab.org/) (you can see at the beginning, the estimated value is converging to the truth despite wrong initial value):

2. [ukf_example2_imu](ukf_example2_imu) (4 state, 3 input, 6 output): **206 us** to compute one iteration (single precision math) or **246 us** (double precision). The result, displayed by [Processing](https://processing.org/) script based on [FreeIMU project](http://www.varesano.net/files/FreeIMU-20121122_1126.zip):

You can also see the video in the [ukf_example2_imu](ukf_example2_imu) folder.

# Closing Remark
I hope you can test & validate my result or inform me if there are some bugs / mathematical error you encounter along the way! (or if you notice some grammar error in the documentation).

I published the code under CC0 license, effectively placed the code on public domain. But it will be great if you can tell me if you use the code, for what/why. That means a lot to me and give me motivation to expand the work (⌒▽⌒)

mime_type: text/x-c++


text:
/***********************************************************************************************************************
 *  Class for Discrete Unscented Kalman Filter
 *  Ref: Van der. Merwe, .. (2004). Sigma-Point Kalman Filters for Probabilistic Inference in Dynamic
 *      State-Space Models (Ph.D. thesis). Oregon Health & Science University.
 *
 *  The system to be estimated is defined as a discrete nonlinear dynamic dystem:
 *              x(k+1) = f[x(k), u(k)] + v(k)           ; x = Nx1,    u = Mx1
 *              y(k)   = h[x(k), u(k)] + n(k)           ; y = Zx1
 *
 *        Where:
 *          x(k) : State Variable at time-k                          : Nx1
 *          y(k) : Measured output at time-k                         : Zx1
 *          u(k) : System input at time-k                            : Mx1
 *          v(k) : Process noise, AWGN assumed, w/ covariance  Rv    : Nx1
 *          n(k) : Measurement noise, AWGN assumed, w/ covariance Rn : Nx1
 *
 *          f(..), h(..) is a nonlinear transformation of the system to be estimated.
 *
 ***********************************************************************************************************************
 *      Unscented Kalman Filter algorithm:
 *          Initialization:
 *              P(k=0|k=0) = Identity matrix * covariant(P(k=0)), typically initialized with some big number.
 *              x(k=0|k=0) = Expected value of x at time-0 (i.e. x(k=0)), typically set to zero.
 *              Rv, Rn     = Covariance matrices of process & measurement. As this implementation
 *                             the noise as AWGN (and same value for every variable), this is set
 *                             to Rv=diag(RvInit,...,RvInit) and Rn=diag(RnInit,...,RnInit).
 *              Wc, Wm     = First order & second order weight, respectively.
 *              alpha, beta, kappa, gamma = scalar constants.
 *
 *              lambda = (alpha^2)*(N+kappa)-N,         gamma = sqrt(N+alpha)           ...{UKF_1}
 *              Wm = [lambda/(N+lambda)         1/(2(N+lambda)) ... 1/(2(N+lambda))]    ...{UKF_2}
 *              Wc = [Wm(0)+(1-alpha(^2)+beta)  1/(2(N+lambda)) ... 1/(2(N+lambda))]    ...{UKF_3}
 *
 *
 *          UKF Calculation (every sampling time):
 *              Calculate the Sigma Point:
 *                  Xs(k-1) = [x(k-1) ... x(k-1)]            ; Xs(k-1) = NxN
 *                  GPsq = gamma * sqrt(P(k-1))
 *                  XSigma(k-1) = [x(k-1) Xs(k-1)+GPsq Xs(k-1)-GPsq]                    ...{UKF_4}
 *
 *
 *              Unscented Transform XSigma [f,XSigma,u,Rv] -> [x,XSigma,P,DX]:
 *                  XSigma(k) = f(XSigma(k-1), u(k-1))                                  ...{UKF_5a}
 *
 *                  x(k|k-1) = sum(Wm(i) * XSigma(k)(i))    ; i = 1 ... (2N+1)          ...{UKF_6a}
 *
 *                  DX = XSigma(k)(i) - Xs(k)   ; Xs(k) = [x(k|k-1) ... x(k|k-1)]
 *                                              ; Xs(k) = Nx(2N+1)                      ...{UKF_7a}
 *
 *                  P(k|k-1) = sum(Wc(i)*DX*DX') + Rv       ; i = 1 ... (2N+1)          ...{UKF_8a}
 *
 *
 *              Unscented Transform YSigma [h,XSigma,u,Rn] -> [y_est,YSigma,Py,DY]:
 *                  YSigma(k) = h(XSigma(k), u(k|k-1))      ; u(k|k-1) = u(k)           ...{UKF_5b}
 *
 *                  y_est(k) = sum(Wm(i) * YSigma(k)(i))    ; i = 1 ... (2N+1)          ...{UKF_6b}
 *
 *                  DY = YSigma(k)(i) - Ys(k)   ; Ys(k) = [y_est(k) ... y_est(k)]
 *                                              ; Ys(k) = Zx(2N+1)                      ...{UKF_7b}
 *
 *                  Py(k) = sum(Wc(i)*DY*DY') + Rn          ; i = 1 ... (2N+1)          ...{UKF_8b}
 *
 *
 *              Calculate Cross-Covariance Matrix:
 *                  Pxy(k) = sum(Wc(i)*DX*DY(i))            ; i = 1 ... (2N+1)          ...{UKF_9}
 *
 *
 *              Calculate the Kalman Gain:
 *                  K           = Pxy(k) * (Py(k)^-1)                                   ...{UKF_10}
 *
 *
 *              Update the Estimated State Variable:
 *                  x(k|k)      = x(k|k-1) + K * (y(k) - y_est(k))                      ...{UKF_11}
 *
 *
 *              Update the Covariance Matrix:
 *                  P(k|k)      = P(k|k-1) - K*Py(k)*K'                                 ...{UKF_12}
 *
 *
 *        *Additional Information:
 *              - Dengan asumsi masukan plant ZOH, u(k) = u(k|k-1),
 *                  Dengan asumsi tambahan observer dijalankan sebelum pengendali, u(k|k-1) = u(k-1),
 *                  sehingga u(k) [untuk perhitungan kalman] adalah nilai u(k-1) [dari pengendali].
 *              - Notasi yang benar adalah u(k|k-1), tapi disini menggunakan notasi u(k) untuk
 *                  menyederhanakan penulisan rumus.
 *              - Pada contoh di atas X~(k=0|k=0) = [0]. Untuk mempercepat konvergensi bisa digunakan
 *                  informasi plant-spesific. Misal pada implementasi Kalman Filter untuk sensor
 *                  IMU (Inertial measurement unit) dengan X = [quaternion], dengan asumsi IMU
 *                  awalnya menghadap ke atas tanpa rotasi, X~(k=0|k=0) = [1, 0, 0, 0]'
 *
 * See https://github.com/pronenewbits for more!
 **********************************************************************************************************************/
#include "ukf.h"

UKF::UKF(const Matrix& XInit, const Matrix& PInit, const Matrix& Rv, const Matrix& Rn,
         bool (*bNonlinearUpdateX)(Matrix&, const Matrix&, const Matrix&),
         bool (*bNonlinearUpdateY)(Matrix&, const Matrix&, const Matrix&))
{
    /* Initialization:
     *  P(k=0|k=0) = Identity matrix * covariant(P(k=0)), typically initialized with some big number.
     *  x(k=0|k=0) = Expected value of x at time-0 (i.e. x(k=0)), typically set to zero.
     *  Rv, Rn     = Covariance matrices of process & measurement. As this implementation
     *                the noise as AWGN (and same value for every variable), this is set
     *                to Rv=diag(RvInit,...,RvInit) and Rn=diag(RnInit,...,RnInit).
     */
    this->X_Est = XInit;
    this->P     = PInit;
    this->Rv    = Rv;
    this->Rn    = Rn;
    this->bNonlinearUpdateX = bNonlinearUpdateX;
    this->bNonlinearUpdateY = bNonlinearUpdateY;

    /* Van der. Merwe, .. (2004). Sigma-Point Kalman Filters for Probabilistic Inference in Dynamic State-Space Models
     * (Ph.D. thesis). Oregon Health & Science University. Page 6:
     *
     * where λ = α2(L+κ)−L is a scaling parameter. α determines the spread of the sigma points around ̄x and is usually
     * set to a small positive value (e.g. 1e−2 ≤ α ≤ 1). κ is a secondary scaling parameter which is usually set to
     * either 0 or 3−L (see [45] for details), and β is an extra degree of freedom scalar parameter used to
     * incorporate any extra prior knowledge of the distribution of x (for Gaussian distributions, β = 2 is optimal).
     */
    float_prec _alpha   = 1e-2;
    float_prec _k       = 0.0;
    float_prec _beta    = 2.0;

    /* lambda = (alpha^2)*(N+kappa)-N,         gamma = sqrt(N+alpha)            ...{UKF_1} */
    float_prec _lambda  = (_alpha*_alpha)*(SS_X_LEN+_k) - SS_X_LEN;
    Gamma = sqrt((SS_X_LEN + _lambda));

    /* Wm = [lambda/(N+lambda)         1/(2(N+lambda)) ... 1/(2(N+lambda))]     ...{UKF_2} */
    Wm[0][0] = _lambda/(SS_X_LEN + _lambda);
    for (int16_t _i = 1; _i < Wm.i16getCol(); _i++) {
        Wm[0][_i] = 0.5/(SS_X_LEN + _lambda);
    }

    /* Wc = [Wm(0)+(1-alpha(^2)+beta)  1/(2(N+lambda)) ... 1/(2(N+lambda))]     ...{UKF_3} */
    Wc = Wm;
    Wc[0][0] = Wc[0][0] + (1.0-(_alpha*_alpha)+_beta);
}

void UKF::vReset(const Matrix& XInit, const Matrix& PInit, const Matrix& Rv, const Matrix& Rn)
{
    this->X_Est = XInit;
    this->P     = PInit;
    this->Rv    = Rv;
    this->Rn    = Rn;
}

bool UKF::bUpdate(const Matrix& Y, const Matrix& U)
{
    /* Run once every sampling time */

    /* XSigma(k-1) = [x(k-1) Xs(k-1)+GPsq Xs(k-1)-GPsq]                     ...{UKF_4}  */
    if (!bCalculateSigmaPoint()) {
        return false;
    }

    /* Unscented Transform XSigma [f,XSigma,u,Rv] -> [x,XSigma,P,DX]:       ...{UKF_5a} - {UKF_8a} */
    if (!bUnscentedTransform(X_Est, X_Sigma, P, DX, bNonlinearUpdateX, X_Sigma, U, Rv)) {
        return false;
    }

    /* Unscented Transform YSigma [h,XSigma,u,Rn] -> [y_est,YSigma,Py,DY]:  ...{UKF_5b} - {UKF_8b} */
    if (!bUnscentedTransform(Y_Est, Y_Sigma, Py, DY, bNonlinearUpdateY, X_Sigma, U, Rn)) {
        return false;
    }

    /* Calculate Cross-Covariance Matrix:
     *  Pxy(k) = sum(Wc(i)*DX*DY(i))            ; i = 1 ... (2N+1)          ...{UKF_9}
     */
    for (int16_t _i = 0; _i < DX.i16getRow(); _i++) {
        for (int16_t _j = 0; _j < DX.i16getCol(); _j++) {
            DX[_i][_j] *= Wc[0][_j];
        }
    }
    Pxy = DX * (DY.Transpose());

    /* Calculate the Kalman Gain:
     *  K           = Pxy(k) * (Py(k)^-1)                                   ...{UKF_10}
     */
    Matrix PyInv(Py.Invers());
    if (!PyInv.bMatrixIsValid()) {
        return false;
    }
    Gain = Pxy * PyInv;

    /* Update the Estimated State Variable:
     *  x(k|k)      = x(k|k-1) + K * (y(k) - y_est(k))                      ...{UKF_11}
     */
    Err = Y - Y_Est;
    X_Est = X_Est + (Gain*Err);

    /* Update the Covariance Matrix:
     *  P(k|k)      = P(k|k-1) - K*Py(k)*K'                                 ...{UKF_12}
     */
    P = P - (Gain * Py * Gain.Transpose());

    return true;
}

bool UKF::bCalculateSigmaPoint(void)
{
    /* Xs(k-1) = [x(k-1) ... x(k-1)]            ; Xs(k-1) = NxN
     * GPsq = gamma * sqrt(P(k-1))
     * XSigma(k-1) = [x(k-1) Xs(k-1)+GPsq Xs(k-1)-GPsq]                     ...{UKF_4}
     */
    /* Use Cholesky Decomposition to compute sqrt(P) */
    P_Chol = P.CholeskyDec();
    if (!P_Chol.bMatrixIsValid()) {
        /* System Fail */
        return false;
    }
    P_Chol = P_Chol * Gamma;

    /* Xs(k-1) = [x(k-1) ... x(k-1)]            ; Xs(k-1) = NxN */
    Matrix _Y(SS_X_LEN, SS_X_LEN);
    for (int16_t _i = 0; _i < SS_X_LEN; _i++) {
        _Y = _Y.InsertVector(X_Est, _i);
    }

    X_Sigma.vSetToZero();
    /* XSigma(k-1) = [x(k-1) 0 0] */
    X_Sigma = X_Sigma.InsertVector(X_Est, 0);
    /* XSigma(k-1) = [x(k-1) Xs(k-1)+GPsq  0] */
    X_Sigma = X_Sigma.InsertSubMatrix((_Y + P_Chol), 0, 1);
    /* XSigma(k-1) = [x(k-1) Xs(k-1)+GPsq Xs(k-1)-GPsq] */
    X_Sigma = X_Sigma.InsertSubMatrix((_Y - P_Chol), 0, (1+SS_X_LEN));

    return true;
}

bool UKF::bUnscentedTransform(Matrix& Out, Matrix& OutSigma, Matrix& P, Matrix& DSig,
                              bool (*_vFuncNonLinear)(Matrix& xOut, const Matrix& xInp, const Matrix& U),
                              const Matrix& InpSigma, const Matrix& InpVector,
                              const Matrix& _CovNoise)
{
    /* XSigma(k) = f(XSigma(k-1), u(k-1))                                  ...{UKF_5a}  */
    /* x(k|k-1) = sum(Wm(i) * XSigma(k)(i))    ; i = 1 ... (2N+1)          ...{UKF_6a}  */
    Out.vSetToZero();
    for (int16_t _j = 0; _j < InpSigma.i16getCol(); _j++) {
        /* Transform the column submatrix of sigma-points input matrix (InpSigma) */
        Matrix _AuxSigma1(InpSigma.i16getRow(), 1);
        Matrix _AuxSigma2(OutSigma.i16getRow(), 1);
        for (int16_t _i = 0; _i < InpSigma.i16getRow(); _i++) {
            _AuxSigma1[_i][0] = InpSigma[_i][_j];
        }
        if (!_vFuncNonLinear(_AuxSigma2, _AuxSigma1, InpVector)) {
            /* Somehow the transformation function is failed, propagate the error */
            return false;
        }

        /* Combine the transformed vector to construct sigma-points output matrix (OutSigma) */
        OutSigma = OutSigma.InsertVector(_AuxSigma2, _j);

        /* Calculate x(k|k-1) as weighted mean of OutSigma */
        _AuxSigma2 = _AuxSigma2 * Wm[0][_j];
        Out = Out + _AuxSigma2;
    }

    /* DX = XSigma(k)(i) - Xs(k)   ; Xs(k) = [x(k|k-1) ... x(k|k-1)]
     *                             ; Xs(k) = Nx(2N+1)                      ...{UKF_7a}  */
    Matrix _AuxSigma1(OutSigma.i16getRow(), OutSigma.i16getCol());
    for (int16_t _j = 0; _j < OutSigma.i16getCol(); _j++) {
        _AuxSigma1 = _AuxSigma1.InsertVector(Out, _j);
    }
    DSig = OutSigma - _AuxSigma1;

    /* P(k|k-1) = sum(Wc(i)*DX*DX') + Rv       ; i = 1 ... (2N+1)          ...{UKF_8a}  */
    _AuxSigma1 = DSig;
    for (int16_t _i = 0; _i < DSig.i16getRow(); _i++) {
        for (int16_t _j = 0; _j < DSig.i16getCol(); _j++) {
            _AuxSigma1[_i][_j] *= Wc[0][_j];
        }
    }
    P = (_AuxSigma1 * (DSig.Transpose())) + _CovNoise;

    return true;
}


text:
/*************************************************************************************************************
 *  Template project for Discrete Unscented Kalman Filter library
 *
 * See https://github.com/pronenewbits for more!
 ************************************************************************************************************/
#include <Wire.h>
#include <elapsedMillis.h>
#include "konfig.h"
#include "matrix.h"
#include "ukf.h"

/* ============================================ UKF Variables/function declaration ============================================ */
/* Just example; in konfig.h:
 *  SS_X_LEN = 2
 *  SS_Z_LEN = 1
 *  SS_U_LEN = 1
 */
/* UKF initialization constant -------------------------------------------------------------------------------------- */
#define P_INIT      (10.)
#define Rv_INIT     (1e-6)
#define Rn_INIT     (0.0015)
/* P(k=0) variable -------------------------------------------------------------------------------------------------- */
float_prec UKF_PINIT_data[SS_X_LEN*SS_X_LEN] = {P_INIT, 0,
                                                0,      P_INIT};
Matrix UKF_PINIT(SS_X_LEN, SS_X_LEN, UKF_PINIT_data);
/* Rv constant ------------------------------------------------------------------------------------------------------ */
float_prec UKF_RVINIT_data[SS_X_LEN*SS_X_LEN] = {Rv_INIT, 0,
                                                 0,      Rv_INIT};
Matrix UKF_RvINIT(SS_X_LEN, SS_X_LEN, UKF_RVINIT_data);
/* Rn constant ------------------------------------------------------------------------------------------------------ */
float_prec UKF_RNINIT_data[SS_Z_LEN*SS_Z_LEN] = {Rn_INIT};
Matrix UKF_RnINIT(SS_Z_LEN, SS_Z_LEN, UKF_RNINIT_data);
/* Nonlinear & linearization function ------------------------------------------------------------------------------- */
bool Main_bUpdateNonlinearX(Matrix& X_Next, const Matrix& X, const Matrix& U);
bool Main_bUpdateNonlinearY(Matrix& Y, const Matrix& X, const Matrix& U);
/* UKF variables ---------------------------------------------------------------------------------------------------- */
Matrix X(SS_X_LEN, 1);
Matrix Y(SS_Z_LEN, 1);
Matrix U(SS_U_LEN, 1);
/* UKF system declaration ------------------------------------------------------------------------------------------- */
UKF UKF_IMU(X, UKF_PINIT, UKF_RvINIT, UKF_RnINIT, Main_bUpdateNonlinearX, Main_bUpdateNonlinearY);

/* ========================================= Auxiliary variables/function declaration ========================================= */
elapsedMillis timerLed, timerUKF;
uint64_t u64compuTime;
char bufferTxSer[100];

void setup() {
    /* serial to display data */
    Serial.begin(115200);
    while(!Serial) {}

    X.vSetToZero();
    UKF_IMU.vReset(X, UKF_PINIT, UKF_RvINIT, UKF_RnINIT);
}

void loop() {
    if (timerUKF >= SS_DT_MILIS) {
        timerUKF = 0;

        /* ================== Read the sensor data / simulate the system here ================== */
        /* ... */
        /* ------------------ Read the sensor data / simulate the system here ------------------ */

        /* ============================= Update the Kalman Filter ============================== */
        u64compuTime = micros();
        if (!UKF_IMU.bUpdate(Y, U)) {
            X.vSetToZero();
            UKF_IMU.vReset(X, UKF_PINIT, UKF_RvINIT, UKF_RnINIT);
            Serial.println("Whoop ");
        }
        u64compuTime = (micros() - u64compuTime);
        /* ----------------------------- Update the Kalman Filter ------------------------------ */

        /* =========================== Print to serial (for plotting) ========================== */
        #if (1)
            /* Print: Computation time, X_Est[0] */
            snprintf(bufferTxSer, sizeof(bufferTxSer)-1, "%.3f %.3f ", ((float)u64compuTime)/1000., UKF_IMU.GetX()[0][0]);
            Serial.print(bufferTxSer);
        #endif
        Serial.print('\n');
        /* --------------------------- Print to serial (for plotting) -------------------------- */
    }
}

bool Main_bUpdateNonlinearX(Matrix& X_Next, const Matrix& X, const Matrix& U)
{
    /* Insert the nonlinear update transformation here
     *          x(k+1) = f[x(k), u(k)]
     */

    return true;
}

bool Main_bUpdateNonlinearY(Matrix& Y, const Matrix& X, const Matrix& U)
{
    /* Insert the nonlinear measurement transformation here
     *          y(k)   = h[x(k), u(k)]
     */

    return true;
}

void SPEW_THE_ERROR(char const * str)
{
    #if (SYSTEM_IMPLEMENTATION == SYSTEM_IMPLEMENTATION_PC)
        cout << (str) << endl;
    #elif (SYSTEM_IMPLEMENTATION == SYSTEM_IMPLEMENTATION_EMBEDDED_ARDUINO)
        Serial.println(str);
    #else
        /* Silent function */
    #endif
    while(1);
}


text:
/************************************************************************************
 * Class Matrix
 *  See matrix.h for description
 *
 * See https://github.com/pronenewbits for more!
 *************************************************************************************/

#include "matrix.h"

/* There's nothing here yet! */