I'm using the PubSub to Elastic Dataflow template and I need to do some enrichment, like XML parsing.
I have two issues:
Any guidance would be appreciated.
Thanks
I was almost there. I wasnt building the single js file properly.
src/index.js
var convert = require("xml-js");
module.exports.data_process = function (inJson) {
var xml = "<XML CONTENT>";
var result1 = convert.xml2json(xml, { compact: true, spaces: 4 });
var parsed = JSON.parse(result1);
var some_field = parsed.some.field
var obj = JSON.parse(inJson);
obj.some_field = some_field;
return JSON.stringify(obj);
};
webpack.config.js
const path = require("path");
module.exports = {
entry: "./src/index.ts",
mode: "production",
output: {
filename: "main.js",
path: path.resolve(__dirname, "dist"),
library: "xmllib",
},
};
build webpack
npx webpack --config webpack.config.js
Then add to the end of main.js
function process(d) {
c = xmllib.data_process(d);
return c;
}
Now you can use "process" as your dataflow udf function name.
Solution based on this article: https://blog.salrashid.dev/articles/2020/bq-udf-protobuf/