Decode vs. encode in node.js

[dancesWithCycles]

Hi folks,
I hope this finds you well. I hope I am blind or misunderstood the bindings version for node.js. I can get a decode example working but not for encoding. Here are the respective decoding and encoding source files. Can anyone open up my eyes please?

Decoding:

const GtfsRealtimeBindings = require('gtfs-realtime-bindings');
const debug = require('debug')('gtfs');
const fs = require("fs");

fs.open('./vehicle_position_another.pb', 'r', function(status, fd) {
    if (status) {
        debug('status: %s',status.message);
        return;
    }else{
        // fs.readFile takes the file path and the callback
        const buffer=fs.readFileSync('./vehicle_position_another.pb', (err, data) => {
            // if there's an error, log it and return
            if (err) {
                debug('error: %s',err)
                return
            }
        })
        // Print the string representation of the data
        if(buffer){
            // decode
            var feed = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(buffer);
            if(feed){
                debug('feed: %s',feed)
                debug('feed.toString(): %s',feed.toString())
                let header=feed.header;
                if(header){
                    debug('header: %s',header)
                    let gtfsRealtimeVersion=header.gtfsRealtimeVersion;
                    if(gtfsRealtimeVersion){
                        debug('gtfsRealtimeVersion: %s',gtfsRealtimeVersion)
                    }
                }
                feed.entity.forEach(function(entity) {
                    if (entity.trip_update) {
                        debug('trip update')
                    }else if(entity.vehicle){
                        debug('vehicle position')
                        debug('entity to string: %s',entity.vehicle.toString())
                    }else if(entity.alert){
                        debug('alert')
                    }else{
                        debug('entity unknown')
                    }
                });
            
                // Verify the payload if necessary (i.e. when possibly incomplete or invalid)
                let errMsg = GtfsRealtimeBindings.transit_realtime.FeedMessage.verify(feed);
                if (errMsg){
                    debug('msg invalid')
                    throw Error(errMsg);
                }else{
                    debug('msg valid')
                }

                feed.header.gtfsRealtimeVersion='2.0';

                // Verify the payload if necessary (i.e. when possibly incomplete or invalid)
                errMsg = GtfsRealtimeBindings.transit_realtime.FeedMessage.verify(feed);
                if (errMsg){
                    debug('msg invalid')
                    throw Error(errMsg);
                }else{
                    debug('msg valid')
                }

                const bufEnc = GtfsRealtimeBindings.transit_realtime.FeedMessage.encode(feed);
                debug('bufEnc: %s',bufEnc)
                debug('bufEnc.toString(): %s',bufEnc.toString())
            }else{
                debug('feed unavailabe')
            }
        }else{
            debug('buffer unavailable')
        }
    }
});

Encoding:

const GtfsRealtimeBindings = require('gtfs-realtime-bindings');
const debug = require('debug')('gtfs');

let msgFeedHdr={
    gtfs_realtime_version:"2.0"
}

var feedHeader = GtfsRealtimeBindings.transit_realtime.FeedHeader.create(msgFeedHdr); // or use .fromObject if conversion is necessary
debug('feedHeader created')
debug('feedHeader: %s',feedHeader)

var errMsg = GtfsRealtimeBindings.transit_realtime.FeedHeader.verify(feedHeader);
if (errMsg){
    debug('feedHeader invalid')
    throw Error(errMsg);
}else{
    debug('feedHeader valid')
}

let msgFeedMsg={
    header:{feedHeader}
}

var feedMessage = GtfsRealtimeBindings.transit_realtime.FeedMessage.create(msgFeedMsg); // or use .fromObject if conversion is necessary
debug('feedMessage created')
debug('feedMessage: %s',feedMessage)

var errMsg = GtfsRealtimeBindings.transit_realtime.FeedMessage.verify(feedMessage);
if (errMsg){
    debug('feedMessage invalid')
    throw Error(errMsg);
}else{
    debug('feedMessage valid')
}

Decoding output:

~/Desktop/sandbox/gtfsRtBindingsDecode$ nodemon index.js 
[nodemon] 2.0.7
[nodemon] to restart at any time, enter `rs`
[nodemon] watching path(s): *.*
[nodemon] watching extensions: js,mjs,json
[nodemon] starting `node index.js`
  gtfs feed: [object Object] +0ms
  gtfs feed.toString(): [object Object] +4ms
  gtfs header: [object Object] +0ms
  gtfs gtfsRealtimeVersion: 1.0 +0ms
  gtfs vehicle position +0ms
  gtfs entity to string: [object Object] +0ms
  gtfs msg valid +1ms
  gtfs msg valid +0ms
  gtfs bufEnc: [object Object] +2ms
  gtfs bufEnc.toString(): [object Object] +1ms
[nodemon] clean exit - waiting for changes before restart

Encoding output:

~/Desktop/sandbox/gtfsRtBindingsEncode$ nodemon index.js 
[nodemon] 2.0.7
[nodemon] to restart at any time, enter `rs`
[nodemon] watching path(s): *.*
[nodemon] watching extensions: js,mjs,json
[nodemon] starting `node index.js`
  gtfs feedHeader created +0ms
  gtfs feedHeader: [object Object] +3ms
  gtfs feedHeader valid +1ms
  gtfs feedMessage created +0ms
  gtfs feedMessage: [object Object] +0ms
  gtfs feedMessage invalid +0ms
/home/user/Desktop/sandbox/gtfsRtBindingsEncode/index.js:31
    throw Error(errMsg);
    ^

Error: header.gtfsRealtimeVersion: string expected
    at Object.<anonymous> (/home/user/Desktop/sandbox/gtfsRtBindingsEncode/index.js:31:11)
    at Module._compile (internal/modules/cjs/loader.js:778:30)
    at Object.Module._extensions..js (internal/modules/cjs/loader.js:789:10)
    at Module.load (internal/modules/cjs/loader.js:653:32)
    at tryModuleLoad (internal/modules/cjs/loader.js:593:12)
    at Function.Module._load (internal/modules/cjs/loader.js:585:3)
    at Function.Module.runMain (internal/modules/cjs/loader.js:831:12)
    at startup (internal/bootstrap/node.js:283:19)
    at bootstrapNodeJSCore (internal/bootstrap/node.js:623:3)
[nodemon] app crashed - waiting for file changes before starting...

Cheers!

[dancesWithCycles]

Hi folks,
my next try ended in a message that is valid according to the library but can not be encoded.

Source code:

const GtfsRealtimeBindings = require('gtfs-realtime-bindings');
const debug = require('debug')('gtfs');

var feedHeader = GtfsRealtimeBindings.transit_realtime.FeedHeader.create(); // or use .fromObject if conversion is necessary
debug('feedHeader created')

feedHeader.gtfs_realtime_version="2.0"

var feedMessage = GtfsRealtimeBindings.transit_realtime.FeedMessage.create(); // or use .fromObject if conversion is necessary
debug('feedMessage created')

feedMessage.header=feedHeader;

var feedEntity = GtfsRealtimeBindings.transit_realtime.FeedEntity.create(); // or use .fromObject if conversion is necessary
debug('feedEntity created')

var vehiclePosition = GtfsRealtimeBindings.transit_realtime.VehiclePosition.create(); // or use .fromObject if conversion is necessary
debug('vehiclePosition created')

var pos = GtfsRealtimeBindings.transit_realtime.Position.create(); // or use .fromObject if conversion is necessary
debug('pos created')

pos.latitude=36;
pos.longitude=-79;

var errMsg = GtfsRealtimeBindings.transit_realtime.Position.verify(pos);
if (errMsg){
    debug('pos invalid')
    throw Error(errMsg);
}else{
    debug('pos valid')
}

vehiclePosition.position=pos;
feedEntity.vehicle=vehiclePosition;
feedMessage.entity=[feedEntity];

var errMsg = GtfsRealtimeBindings.transit_realtime.FeedMessage.verify(feedMessage);
if (errMsg){
    debug('feedMessage invalid')
    throw Error(errMsg);
}else{
    debug('feedMessage valid')
}

var errMsg = GtfsRealtimeBindings.transit_realtime.FeedMessage.encode(feedMessage);
if (errMsg){
    debug('encode error: %s',errMsg.toString())
    throw Error(errMsg);
}else{
    debug('encode success')
}

Error:

[nodemon] restarting due to changes...
[nodemon] starting `node index.js`
  gtfs feedHeader created +0ms
  gtfs feedHeader valid +3ms
  gtfs feedMessage created +1ms
  gtfs feedMessage valid +0ms
  gtfs feedEntity created +0ms
  gtfs vehiclePosition created +0ms
  gtfs vehiclePosition valid +0ms
  gtfs pos created +1ms
  gtfs pos valid +0ms
  gtfs feedEntity valid +0ms
  gtfs feedMessage valid +0ms
  gtfs encode error: [object Object] +2ms
/home/user/Desktop/sandbox/gtfsRtBindingsEncode/index.js:82
    throw Error(errMsg);
    ^

Error: [object Object]
    at Object.<anonymous> (/home/user/Desktop/sandbox/gtfsRtBindingsEncode/index.js:82:11)
    at Module._compile (internal/modules/cjs/loader.js:778:30)
    at Object.Module._extensions..js (internal/modules/cjs/loader.js:789:10)
    at Module.load (internal/modules/cjs/loader.js:653:32)
    at tryModuleLoad (internal/modules/cjs/loader.js:593:12)
    at Function.Module._load (internal/modules/cjs/loader.js:585:3)
    at Function.Module.runMain (internal/modules/cjs/loader.js:831:12)
    at startup (internal/bootstrap/node.js:283:19)
    at bootstrapNodeJSCore (internal/bootstrap/node.js:623:3)
[nodemon] app crashed - waiting for file changes before starting...

Do you guys have any clue what is the reason for the error?

Cheers!

[dancesWithCycles]

Hi folks,
I found the error and solution myself. Anyhow, I am still wondering how can I verify that encoding finished successful without decoding the buffer and looking at the message content. Does anyone has a little advice for me?

Here is the code example:

const GtfsRealtimeBindings = require('gtfs-realtime-bindings');
const debug = require('debug')('gtfs');

var feedHeader = GtfsRealtimeBindings.transit_realtime.FeedHeader.create({
    gtfsRealtimeVersion:'2.0'
});

var pos = GtfsRealtimeBindings.transit_realtime.Position.create({
    latitude:36,longitude:-79});

var vehiclePosition = GtfsRealtimeBindings.transit_realtime.VehiclePosition.create({
position:pos});

var feedEntity = GtfsRealtimeBindings.transit_realtime.FeedEntity.create({
    id:'uuid-foo-bar',vehicle:vehiclePosition
});

var feedMessage = GtfsRealtimeBindings.transit_realtime.FeedMessage.create({
    header:feedHeader,
    entity:[feedEntity]
});

var errMsg = GtfsRealtimeBindings.transit_realtime.FeedMessage.verify(feedMessage);
if (errMsg){
    debug('feedMessage invalid')
    throw Error(errMsg);
}else{
    debug('feedMessage valid')
}

debug("JSON: %s", JSON.stringify(feedMessage));

var encodedFeedMsg = GtfsRealtimeBindings.transit_realtime.FeedMessage.encode(feedMessage).finish();
debug('feedMessage encoded')
var decodedFeedMsg = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(encodedFeedMsg);
debug('feedMessage decoded')
debug("JSON: %s", JSON.stringify(decodedFeedMsg));

And here is the respective CLI output:

[nodemon] starting `node index.js`
  gtfs feedMessage valid +0ms
  gtfs JSON: {"header":{"gtfsRealtimeVersion":"2.0"},"entity":[{"id":"uuid-foo-bar","vehicle":{"position":{"latitude":36,"longitude":-79}}}]} +4ms
  gtfs feedMessage encoded +3ms
  gtfs feedMessage decoded +1ms
  gtfs JSON: {"header":{"gtfsRealtimeVersion":"2.0"},"entity":[{"id":"uuid-foo-bar","vehicle":{"position":{"latitude":36,"longitude":-79}}}]} +0ms
[nodemon] clean exit - waiting for changes before restart

[jameslinjl]

@dancesWithCycles Protobufs are simply a way to serialize data (typically into a packed, binary format), so a successful encoding output is only verifiable by performing a corresponding decode on that encoded message. If you wanted to ensure that the output is truly language and platform neutral, you could take the resulting output and deserialize it using another language. The expectation is that the encoded message could still be decoded.

[jameslinjl]

proto3 version does have a canonical JSON format (as opposed to binary) for serialization, which can help with debugging and triaging these sorts of issues via manual human inspection, but GTFS-Realtime currently uses the proto2 version, so that isn't applicable here