From 20c467bf488f1967e972fade741588257bc56565 Mon Sep 17 00:00:00 2001 From: frabad Date: Fri, 6 May 2022 17:33:17 +0200 Subject: [PATCH] =?UTF-8?q?ajout=20des=20resources=20modifi=C3=A9es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- LittleBock/data/0.html | 10 +++++ LittleBock/data/1.html | 48 ++++++++++++++++++++++++ LittleBock/extract.nim | 40 ++++++++++++++++++++ LittleBock/extract.py | 37 +++++++++++++++++++ TestServer/go.mod | 9 +++++ TestServer/main.go | 83 ++++++++++++++++++++++++++++++++++++++++++ flatten/unixtimes.nim | 19 ++++++++++ flatten/unixtimes.py | 27 ++++++++++++++ 8 files changed, 273 insertions(+) create mode 100644 LittleBock/data/0.html create mode 100644 LittleBock/data/1.html create mode 100644 LittleBock/extract.nim create mode 100755 LittleBock/extract.py create mode 100644 TestServer/go.mod create mode 100644 TestServer/main.go create mode 100644 flatten/unixtimes.nim create mode 100755 flatten/unixtimes.py diff --git a/LittleBock/data/0.html b/LittleBock/data/0.html new file mode 100644 index 0000000..65cb2ea --- /dev/null +++ b/LittleBock/data/0.html @@ -0,0 +1,10 @@ + + + + + + + Brassin #2 - Blond + | Little Bock - Application pour brasseur + + diff --git a/LittleBock/data/1.html b/LittleBock/data/1.html new file mode 100644 index 0000000..e4187cb --- /dev/null +++ b/LittleBock/data/1.html @@ -0,0 +1,48 @@ + + + + + + + Brassin #2 - Blond + | Little Bock - Application pour brasseur + + + + +
+
+ + + diff --git a/LittleBock/extract.nim b/LittleBock/extract.nim new file mode 100644 index 0000000..c8dc75f --- /dev/null +++ b/LittleBock/extract.nim @@ -0,0 +1,40 @@ +import std/os +import std/strutils +import std/htmlparser +import std/xmltree +import std/json +import std/strtabs # To access XmlAttributes + +# extract iSpindel JSON from little bock HTML files + +proc jsparse(data: string): JsonNode = + # get JSON tree + var tree: JsonNode = parseJson(data) + if tree.kind == JObject and tree.hasKey("series"): + tree = tree["series"] + for branch in tree: + for unwanted in ["color","opacity","yAxis","dashStyle"]: + if branch.hasKey(unwanted): branch.delete(unwanted) + return tree + +proc hparse(hfile: string): string = + # parse a named HTML document + let html: XmlNode = loadHtml(hfile) + for hdiv in html.findAll("div"): + if hdiv.attrs.hasKey("id"): + if hdiv.attrs["id"] == "fermentation_log_chart": + let data: string = hdiv.attrs["data-chart-options"] + return pretty(jsparse(data)) + +proc jsextract(hfile: string): string = + # run the parsers + result = "None" + var js: string = hparse(hfile) + if not isEmptyOrWhitespace(js): + let jsfile: string = hfile.changeFileExt("json") + result = jsfile + writeFile(jsfile, js) + +for hfile in walkFiles("*.html"): + echo hfile & " >> " & jsextract(hfile) + diff --git a/LittleBock/extract.py b/LittleBock/extract.py new file mode 100755 index 0000000..49c845e --- /dev/null +++ b/LittleBock/extract.py @@ -0,0 +1,37 @@ +#!/usr/bin/python3 +""" +extracteur de données iSpindel JSON entreposées par Little Bock en HTML +""" +import json +try: + import lxml.html as LX +except ModuleNotFoundError as e: + import sys + print("Le module 'lxml' est nécessaire.\n http://pypi.org/lxml") + sys.exit() +import pathlib + +def proc(path): + s, js = None, None + h = LX.parse(path.name) + x = h.xpath("//*[@id='fermentation_log_chart']") + if x: s = x[0].get('data-chart-options') + if s: + js = json.dumps( + json.loads(s).pop('series'), indent=4, sort_keys=True) or None + if js: + p = path.with_suffix('.json') + with open(p,'w') as f: + f.write(js) + print(f"INFO: extraction des données dans {p.name}.") + +if __name__ == "__main__": + here = pathlib.Path.cwd() + hdocs = tuple(here.glob("*.html")) + if len(hdocs) == 0: + print("Aucun fichier HTML ('.html') trouvé.") + for i in hdocs: + if i.exists() and i.stat().st_size > 0: + proc(i) + + diff --git a/TestServer/go.mod b/TestServer/go.mod new file mode 100644 index 0000000..29d60cd --- /dev/null +++ b/TestServer/go.mod @@ -0,0 +1,9 @@ +module testserver + +go 1.16 + +require ( + github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 + github.com/fatih/color v1.13.0 // indirect + github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 // indirect +) diff --git a/TestServer/main.go b/TestServer/main.go new file mode 100644 index 0000000..381d027 --- /dev/null +++ b/TestServer/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "flag" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net" + "os" + "runtime" + "net/http" + "strings" + "github.com/TylerBrock/colorjson" +) + +var port string = "8080" + +func getIp() string { + addrs, err := net.InterfaceAddrs() + if err != nil { + panic(err) + } + return addrs[1].String() +} + +func getInfo() { + var ip string = strings.Split(getIp(), "/")[0] + dir, _ := os.Getwd() + var ss [] string + if runtime.GOOS == "windows" { + ss = strings.Split(dir, "\\") + } else { + ss = strings.Split(dir, "/") + } + var path string = strings.Join(ss, "/") + + fmt.Println("\n", + "==================================\n", + "iSpindle test server configuration\n", + "==================================\n", + fmt.Sprintf("\n address : %s",ip), + fmt.Sprintf("\n port : %s",port), + fmt.Sprintf("\n path : %s",path), + "\n method : HTTP\n") +} + +func main() { + flag.StringVar(&port, "p", port, "port to use") + flag.Usage = func() { + flag.PrintDefaults() // prints default usage + } + flag.Parse() + var infoArg bool + for _, a := range os.Args[1:] { + if a == "info" { infoArg = true } + } + if infoArg { getInfo() + } else { + http.HandleFunc("/", serve) + log.Fatal(http.ListenAndServe(":"+port, nil)) + } +} + +func serve(w http.ResponseWriter, r *http.Request) { + var obj map[string]interface{} + fmt.Printf("%+v\n", r) + str, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Fatal(err) + } + err = json.Unmarshal([]byte(str), &obj) + if err != nil { + log.Fatal(err) + } + f := colorjson.NewFormatter() + f.Indent = 4 + s, _ := f.Marshal(obj) + fmt.Println(string(s)) + + fmt.Fprint(w, "ok") +} + diff --git a/flatten/unixtimes.nim b/flatten/unixtimes.nim new file mode 100644 index 0000000..4e32d59 --- /dev/null +++ b/flatten/unixtimes.nim @@ -0,0 +1,19 @@ +import std/strformat +import std/times +import strutils + +let timestamps: seq[int64] = @[ + 161252911000, + 162352411900, + 1646927061000, + 166691206100000 +] + +echo "formatted UNIX dates" +for i in timestamps: + var s: string = $i + if len(s) > 10: s = s[0 .. 9] + let datetime = times.fromUnix(parseInt(s)) + let datetime_fmt = datetime.format("yyyy-MM-dd HH:mm:ss") + echo &" {s} : {datetime_fmt}" + diff --git a/flatten/unixtimes.py b/flatten/unixtimes.py new file mode 100755 index 0000000..2e590cf --- /dev/null +++ b/flatten/unixtimes.py @@ -0,0 +1,27 @@ +#!/usr/bin/python3 + +import datetime + +"""""" + +TIMESTAMPS = ( + 161252911000, + 162352411900, + 1646927061000, + 166691206100000, +) + +def from_stamp(i: int) -> str: + dt = str(i) + if len(dt)>10: + dt = dt[:10] + sdt = str(datetime.datetime.fromtimestamp(int(dt))) + print(f" {dt} : {sdt}") + return sdt + +if __name__ == "__main__": + print("formatted UNIX dates") + for i in TIMESTAMPS: + from_stamp(i) + +