[HELP-12977] [Fiware-tech-help] IMPACT: Delivering CartoDb events through Cygnus - JIRA

Details

Type: extRequest
Status: Closed
Priority: Major
Resolution: Done
Fix Version/s: 2021
Component/s: FIWARE-TECH-HELP
Labels:
None

Sender Email:
peter@familiohq.com,jsalvachua@dit.upm.es,jira-help-desk@jira.fiware.org
External Participants:
jsalvachua@dit.upm.es,fiware-tech-help@lists.fiware.org
HD-Chapter:
Data
HD-Enabler:
Cygnus

Description

I've setup Orion and Cygnus so whenever a new entity is created in Orion
it's transferring its data to Carto using the Cygnus CartoDbSink.

However I'm having some issues with this implementation. I'm getting the
data correctly from Orion to Cygnus but when the sink tries to forward
those data to Carto I'm getting this error message (also attached for
better formatting) when trying to persist it in Carto

cygnus_1 | time=2017-11-23T13:40:11.176Z | lvl=INFO |
corr=d3be117c-d053-11e7-ad11-0242ac120009 |
trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ |
comp=cygnus-ngsi | op=getEvents |
msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[312] :
[NGSIRestHandler] Received data
({"subscriptionId":"5a16cd57fde41d458740a0b4","originator":"localhost","contextResponses":[{"contextElement":{"type":"Event","isPattern":"false","id":"getif","attributes":[

{"name":"location","type":"geo:point","value":"79.00555, -148.99808"}

]},"statusCode":{"code":"200","reasonPhrase":"OK"}}]})

cygnus_1 | time=2017-11-23T13:40:11.239Z | lvl=DEBUG |
corr=d3be117c-d053-11e7-ad11-0242ac120009 |
trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ |
comp=cygnus-ngsi | op=getEvents |
msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[320] :
[NGSIRestHandler] Parsed NotifyContextRequest:
{"subscriptionId":"5a16cd57fde41d458740a0b4","originator":"localhost","contextResponses":[{"contextElement":{"id":"getif","type":"Event","isPattern":"false","attributes":[

{"name":"location","type":"geo:point","value":"79.00555, -148.99808","metadatas":[]}

]},"statusCode":{"code":"200","reasonPhrase":"OK"}}]}

cygnus_1 | time=2017-11-23T13:40:11.239Z | lvl=DEBUG |
corr=d3be117c-d053-11e7-ad11-0242ac120009 |
trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ |
comp=cygnus-ngsi | op=getEvents |
msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[345] :
[NGSIRestHandler] NGSI event created for ContextElementResponse:
{"contextElement":{"id":"getif","type":"Event","isPattern":"false","attributes":[

{"name":"location","type":"geo:point","value":"79.00555, -148.99808","metadatas":[]}

]},"statusCode":{"code":"200","reasonPhrase":"OK"}}

cygnus_1 | time=2017-11-23T13:40:11.254Z | lvl=INFO |
corr=d3be117c-d053-11e7-ad11-0242ac120009 |
trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ |
comp=cygnus-ngsi | op=persistRawAggregation |
msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink[616] : [cartodb-sink]
Persisting data at NGSICartoDBSink. Schema (default), Table
(x002fxffffgetifxffffx0045vent), Data
(('2017-11-23T13:40:11.241Z','/','getif','Event',ST_SetSRID(ST_MakePoint(79.00555,-148.99808),
4326)))

cygnus_1 | time=2017-11-23T13:40:11.255Z | lvl=ERROR |
corr=d3be117c-d053-11e7-ad11-0242ac120009 |
trans=efab5d17-12c3-4716-8b1b-*15352ec6ae18

srv=default

subsrv=/

comp=cygnus-ngsi

op=run

msg=org.apache.flume.SinkRunner$PollingRunner[160] : Unable to deliver
event. Exception follows.*

cygnus_1 | java.lang.NullPointerException

*cygnus_1 | at
com.telefonica.iot.cygnus.sinks.NGSICartoDBSink.persistRawAggregation(NGSICartoDBSink.java:625)*

*cygnus_1 | at
com.telefonica.iot.cygnus.sinks.NGSICartoDBSink.persistBatch(NGSICartoDBSink.java:417)*

*cygnus_1 | at
com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:558)*

*cygnus_1 | at
com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370)*

*cygnus_1 | at
org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)*

*cygnus_1 | at
org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)*

cygnus_1 | at java.lang.Thread.run(Thread.java:748)

cygnus_1 | java.lang.IllegalStateException: begin() called when
transaction is OPEN!

cygnus_1 | at
com.google.common.base.Preconditions.checkState(Preconditions.java:145)

cygnus_1 | at
org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131)

cygnus_1 | at
com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482)

cygnus_1 | at
com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370)

cygnus_1 | at
org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)

cygnus_1 | at
org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)

cygnus_1 | at java.lang.Thread.run(Thread.java:748)

cygnus_1 | time=2017-11-23T13:40:18.648Z | lvl=DEBUG | corr=N/A

trans=N/A

srv=N/A

subsrv=N/A

comp=cygnus-ngsi

op=run

msg=org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable[126]
: Checking file:/opt/apache-flume/conf/agent.conf for changes

cygnus_1 | time=2017-11-23T13:40:18.654Z | lvl=WARN | corr=N/A

trans=N/A

srv=N/A

subsrv=N/A

comp=cygnus-ngsi

op=<init>

msg=com.telefonica.iot.cygnus.interceptors.CygnusGroupingRules[49] : No
grouping rules have been read. Details:
/usr/cygnus/conf/grouping_rules.conf (No such file or directory)

cygnus_1 | java.lang.IllegalStateException: begin() called when
transaction is OPEN!

cygnus_1 | at
com.google.common.base.Preconditions.checkState(Preconditions.java:145)

cygnus_1 | at
org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131)

cygnus_1 | at
com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482)

cygnus_1 | at
com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370)

cygnus_1 | at
org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)

cygnus_1 | at
org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)

cygnus_1 | at java.lang.Thread.run(Thread.java:748)

cygnus_1 | java.lang.IllegalStateException: begin() called when
transaction is OPEN!

cygnus_1 | at
com.google.common.base.Preconditions.checkState(Preconditions.java:145)

cygnus_1 | at
org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131)

cygnus_1 | at
com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482)

cygnus_1 | at
com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370)

cygnus_1 | at
org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)

cygnus_1 | at
org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)

cygnus_1 | at java.lang.Thread.run(Thread.java:748)

Are you able to help me figure out why I can't send this information to
carto?

–
Peter Theill
CTO
+45 6171 5096 <61%2071%2050%2096>
peter@familiohq.com

Familio
Artillerivej 86
DK-2300 Copenhagen S
https://familiohq.com/

__________________________________________________________________________________________

You can get more information about our cookies and privacy policies on the following links:

Fiware-tech-help mailing list
Fiware-tech-help@lists.fiware.org
https://lists.fiware.org/listinfo/fiware-tech-help

[Created via e-mail received from: Peter Theill <peter@familiohq.com>]

Options
- Sort By Name
- Sort By Date
- Ascending
- Descending
- Download All

Attachments

notification-json-geom.sh

12/Dec/17 12:04 PM

2 kB

Andres Muñoz
cygnus-log.txt

20/Dec/17 5:27 PM

15 kB

FW External User
carto-cygnus-log.txt

23/Nov/17 11:02 AM

13 kB

FW External User

Activity

Descending order - Click to sort in ascending order

Hide

Permalink

Andres Muñoz added a comment - 30/Jan/18 9:56 AM

Hi,

I am still waiting for the user feedback.

Best Regards

Show

Andres Muñoz added a comment - 30/Jan/18 9:56 AM Hi, I am still waiting for the user feedback. Best Regards

Hide

Permalink

Andres Muñoz added a comment - 23/Jan/18 10:21 AM

Hello Peter,

Please, Could you give me some feedback about the tests that I told you to do?

Best Regards,

Show

Andres Muñoz added a comment - 23/Jan/18 10:21 AM Hello Peter, Please, Could you give me some feedback about the tests that I told you to do? Best Regards,

Hide

Permalink

Andres Muñoz added a comment - 15/Jan/18 10:40 AM

Hello Peter,

I am waiting for the results of the test, as soon as you have any feedback please tell me.

Best Regards

Show

Andres Muñoz added a comment - 15/Jan/18 10:40 AM Hello Peter, I am waiting for the results of the test, as soon as you have any feedback please tell me. Best Regards

Hide

Permalink

FW External User added a comment - 09/Jan/18 3:31 PM

Comment by peter@familiohq.com :

Thanks! I'll run the scripts and get back to you.

On Mon, Jan 8, 2018 at 4:22 PM Help-Desk <jira-help-desk@jira.fiware.org>
wrote:

>
>

Show

FW External User added a comment - 09/Jan/18 3:31 PM Comment by peter@familiohq.com : Thanks! I'll run the scripts and get back to you. On Mon, Jan 8, 2018 at 4:22 PM Help-Desk <jira-help-desk@jira.fiware.org> wrote: > >

Hide

Permalink

Andres Muñoz added a comment - 08/Jan/18 4:21 PM

Hello Peter,

I am still checking your issue but I need that you perform some other tests, please once again run the script that I provide you but this time with this 2 more test values :

./notification.sh http://localhost:5050/notify myservice /traffic 40.361 -3.4099

remember that you must execute this script pointing to your cygnus host and port

then tell me if it was successful persisted in the cartodb otherwise give me the output log.

If and only if it fails please try to execute the next command for testing if the cartodb entities are correctly created and available

for inserting data to cartodb

curl -G "https://<my_user>.cartodb.com/api/v2/sql?api_key=<api_key>" --data-urlencode "q=INSERT INTO <table_name> (recvTime, fiwareServicePath, entityId, entityType, <attr_1>, <attr_1>_md, ..., <attr_n>, <attr_n>_md, the_geom) VALUES ('2016-04-19T07:09:53.116Z', '<service_path>', '<entity_id>', '<entity_type>', '<attr_1_value>', '<attr_1_metadata>', ..., '<attr_n_value>', '<attr_n_metadata>', 'ST_SetSRID(ST_MakePoint(<lat>, <lon>), 4326))"

for query this data

curl -G "https://<my_user>.cartodb.com/api/v2/sql?api_key=<api_key>" --data-urlencode "q=SELECT * FROM <table_name>"

I will be waiting for your comments.

Best Regards

Show

Andres Muñoz added a comment - 08/Jan/18 4:21 PM Hello Peter, I am still checking your issue but I need that you perform some other tests, please once again run the script that I provide you but this time with this 2 more test values : ./notification.sh http://localhost:5050/notify myservice /traffic 40.361 -3.4099 remember that you must execute this script pointing to your cygnus host and port then tell me if it was successful persisted in the cartodb otherwise give me the output log. If and only if it fails please try to execute the next command for testing if the cartodb entities are correctly created and available for inserting data to cartodb curl -G "https://<my_user>.cartodb.com/api/v2/sql?api_key=<api_key>" --data-urlencode "q=INSERT INTO <table_name> (recvTime, fiwareServicePath, entityId, entityType, <attr_1>, <attr_1>_md, ..., <attr_n>, <attr_n>_md, the_geom) VALUES ('2016-04-19T07:09:53.116Z', '<service_path>', '<entity_id>', '<entity_type>', '<attr_1_value>', '<attr_1_metadata>', ..., '<attr_n_value>', '<attr_n_metadata>', 'ST_SetSRID(ST_MakePoint(<lat>, <lon>), 4326))" for query this data curl -G "https://<my_user>.cartodb.com/api/v2/sql?api_key=<api_key>" --data-urlencode "q=SELECT * FROM <table_name>" I will be waiting for your comments. Best Regards

Hide

Permalink

FW External User added a comment - 20/Dec/17 6:46 PM

Comment by peter@familiohq.com :

No prob and thanks for looking into it!
On Wed, 20 Dec 2017 at 17.41, Help-Desk <jira-help-desk@jira.fiware.org>
wrote:

>
>

Show

FW External User added a comment - 20/Dec/17 6:46 PM Comment by peter@familiohq.com : No prob and thanks for looking into it! On Wed, 20 Dec 2017 at 17.41, Help-Desk <jira-help-desk@jira.fiware.org> wrote: > >

Hide

Permalink

Andres Muñoz added a comment - 20/Dec/17 5:40 PM

Thanks,

With this test, I can discard something, let me check the code of the sink for trying to locate the trouble, as soon as I have anything I will write to you.

Thanks for your patience

Show

Andres Muñoz added a comment - 20/Dec/17 5:40 PM Thanks, With this test, I can discard something, let me check the code of the sink for trying to locate the trouble, as soon as I have anything I will write to you. Thanks for your patience

Hide

Permalink

FW External User added a comment - 20/Dec/17 5:27 PM

Comment by peter@familiohq.com :

I got it up and running by recompiling the docker image. Might have messed
it up while trying to get it to work. I've attached the log to the mail but
also pasted it below. I've marked the interesting logs with bold and the
error in red.

Let me know if you would like some other log.

cygnus-ngsi_1 |* 2017-12-20T16:16:54.587039854Z
time=2017-12-20T16:16:54.586Z | lvl=INFO |
corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 |
trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ |
comp=cygnus-ngsi | op=getEvents |
msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[312] :
[NGSIRestHandler] Received data ({ "subscriptionId" :
"51c0ac9ed714fb3b37d7d5a8", "originator" : "localhost",
"contextResponses" : [ { "contextElement" : { "attributes" :
[

{ "name" : "temperature", "type" : "centigrade", "value" : "26.5" }

, {
"name" : "the_geom", "type" : "geometry", "value" :
", ", "metadatas": [

{ "name": "location", "type": "string", "value": "WGS84" }

] } ], "type" :
"Car", "isPattern" : "false", "id" : "Car1" },
"statusCode" :

{ "code" : "200", "reasonPhrase" : "OK" }

} ]})*

cygnus-ngsi_1 | 2017-12-20T16:16:54.649653920Z
time=2017-12-20T16:16:54.649Z | lvl=DEBUG |
corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 |
trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ |
comp=cygnus-ngsi | op=getEvents |
msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[320] :
[NGSIRestHandler] Parsed NotifyContextRequest:
{"subscriptionId":"51c0ac9ed714fb3b37d7d5a8","originator":"localhost","contextResponses":[{"contextElement":{"id":"Car1","type":"Car","isPattern":"false","attributes":[

{"name":"temperature","type":"centigrade","value":"26.5","metadatas":[]}

,{"name":"the_geom","type":"geometry","value":",
","metadatas":[

{"name":"location","type":"string","value":"WGS84"}

]}]},"statusCode":{"code":"200","reasonPhrase":"OK"}}]}

cygnus-ngsi_1 | 2017-12-20T16:16:54.649687935Z
time=2017-12-20T16:16:54.649Z | lvl=DEBUG |
corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 |
trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ |
comp=cygnus-ngsi | op=getEvents |
msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[345] :
[NGSIRestHandler] NGSI event created for ContextElementResponse:
{"contextElement":{"id":"Car1","type":"Car","isPattern":"false","attributes":[

{"name":"temperature","type":"centigrade","value":"26.5","metadatas":[]}

,{"name":"the_geom","type":"geometry","value":",
","metadatas":[

{"name":"location","type":"string","value":"WGS84"}

]}]},"statusCode":{"code":"200","reasonPhrase":"OK"}}

cygnus-ngsi_1 | 2017-12-20T16:16:55.257732286Z
time=2017-12-20T16:16:55.257Z | lvl=INFO |
corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 |
trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ |
comp=cygnus-ngsi | op=persistRawAggregation |
msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink[616] : [cartodb-sink]
Persisting data at NGSICartoDBSink. Schema (default), Table
(x002fxffffx0043ar1xffffx0043ar), Data
(('2017-12-20T16:16:54.651Z','/','Car1','Car',ST_SetSRID(ST_MakePoint(,),
4326),'26.5','[]'))

cygnus-ngsi_1 | 2017-12-20T16:16:55.259632910Z
java.lang.NullPointerException

cygnus-ngsi_1 | 2017-12-20T16:16:55.259636430Z at
com.telefonica.iot.cygnus.sinks.NGSICartoDBSink.persistRawAggregation(NGSICartoDBSink.java:625)

cygnus-ngsi_1 | 2017-12-20T16:16:55.259639395Z at
com.telefonica.iot.cygnus.sinks.NGSICartoDBSink.persistBatch(NGSICartoDBSink.java:417)

cygnus-ngsi_1 | 2017-12-20T16:16:55.259643325Z at
com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:558)

cygnus-ngsi_1 | 2017-12-20T16:16:55.259645857Z at
com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370)

cygnus-ngsi_1 | 2017-12-20T16:16:55.259648425Z at
org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)

cygnus-ngsi_1 | 2017-12-20T16:16:55.259651037Z at
org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)

cygnus-ngsi_1 | 2017-12-20T16:16:55.259653547Z at
java.lang.Thread.run(Thread.java:748)

cygnus-ngsi_1 | 2017-12-20T16:17:00.260757599Z
java.lang.IllegalStateException: begin() called when transaction is OPEN!

cygnus-ngsi_1 | 2017-12-20T16:17:00.260761309Z at
com.google.common.base.Preconditions.checkState(Preconditions.java:145)

cygnus-ngsi_1 | 2017-12-20T16:17:00.260764359Z at
org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131)

cygnus-ngsi_1 | 2017-12-20T16:17:00.260767151Z at
com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482)

cygnus-ngsi_1 | 2017-12-20T16:17:00.260769842Z at
com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370)

cygnus-ngsi_1 | 2017-12-20T16:17:00.260784698Z at
org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)

cygnus-ngsi_1 | 2017-12-20T16:17:00.260787712Z at
org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)

cygnus-ngsi_1 | 2017-12-20T16:17:00.260790256Z at
java.lang.Thread.run(Thread.java:748)

subsrv=N/A

comp=cygnus-ngsi

op=run

msg=org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable[126]
: Checking file:/opt/apache-flume/conf/agent.conf for changes

cygnus-ngsi_1 | 2017-12-20T16:17:05.261368680Z
java.lang.IllegalStateException: begin() called when transaction is OPEN!

cygnus-ngsi_1 | 2017-12-20T16:17:05.261372739Z at
com.google.common.base.Preconditions.checkState(Preconditions.java:145)

cygnus-ngsi_1 | 2017-12-20T16:17:05.261376078Z at
org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131)

cygnus-ngsi_1 | 2017-12-20T16:17:05.261379104Z at
com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482)

cygnus-ngsi_1 | 2017-12-20T16:17:05.261382038Z at
com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370)

cygnus-ngsi_1 | 2017-12-20T16:17:05.261384947Z at
org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)

cygnus-ngsi_1 | 2017-12-20T16:17:05.261387928Z at
org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)

cygnus-ngsi_1 | 2017-12-20T16:17:05.261390817Z at
java.lang.Thread.run(Thread.java:748)

cygnus-ngsi_1 | 2017-12-20T16:17:10.262065339Z
java.lang.IllegalStateException: begin() called when transaction is OPEN!

cygnus-ngsi_1 | 2017-12-20T16:17:10.262070038Z at
com.google.common.base.Preconditions.checkState(Preconditions.java:145)

cygnus-ngsi_1 | 2017-12-20T16:17:10.262073562Z at
org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131)

cygnus-ngsi_1 | 2017-12-20T16:17:10.262076685Z at
com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482)

cygnus-ngsi_1 | 2017-12-20T16:17:10.262079595Z at
com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370)

cygnus-ngsi_1 | 2017-12-20T16:17:10.262082479Z at
org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)

cygnus-ngsi_1 | 2017-12-20T16:17:10.262085521Z at
org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)

cygnus-ngsi_1 | 2017-12-20T16:17:10.262100755Z at
java.lang.Thread.run(Thread.java:748)

On Wed, Dec 20, 2017 at 5:07 PM Help-Desk <jira-help-desk@jira.fiware.org>
wrote:

>
>

Show

FW External User added a comment - 20/Dec/17 5:27 PM Comment by peter@familiohq.com : I got it up and running by recompiling the docker image. Might have messed it up while trying to get it to work. I've attached the log to the mail but also pasted it below. I've marked the interesting logs with bold and the error in red. Let me know if you would like some other log. cygnus-ngsi_1 | 2017-12-20T16:16:54.586378023Z time=2017-12-20T16:16:54.586Z | lvl=INFO | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [294] : [NGSIRestHandler] Starting internal transaction (5ec1caf8-1972-4021-ac57-fb78bc48cca9) cygnus-ngsi_1 | * 2017-12-20T16:16:54.587039854Z time=2017-12-20T16:16:54.586Z | lvl=INFO | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [312] : [NGSIRestHandler] Received data ({ "subscriptionId" : "51c0ac9ed714fb3b37d7d5a8", "originator" : "localhost", "contextResponses" : [ { "contextElement" : { "attributes" : [ { "name" : "temperature", "type" : "centigrade", "value" : "26.5" } , { "name" : "the_geom", "type" : "geometry", "value" : ", ", "metadatas": [ { "name": "location", "type": "string", "value": "WGS84" } ] } ], "type" : "Car", "isPattern" : "false", "id" : "Car1" }, "statusCode" : { "code" : "200", "reasonPhrase" : "OK" } } ]})* cygnus-ngsi_1 | 2017-12-20T16:16:54.649653920Z time=2017-12-20T16:16:54.649Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [320] : [NGSIRestHandler] Parsed NotifyContextRequest: {"subscriptionId":"51c0ac9ed714fb3b37d7d5a8","originator":"localhost","contextResponses":[{"contextElement":{"id":"Car1","type":"Car","isPattern":"false","attributes":[ {"name":"temperature","type":"centigrade","value":"26.5","metadatas":[]} ,{"name":"the_geom","type":"geometry","value":", ","metadatas":[ {"name":"location","type":"string","value":"WGS84"} ]}]},"statusCode":{"code":"200","reasonPhrase":"OK"}}]} cygnus-ngsi_1 | 2017-12-20T16:16:54.649687935Z time=2017-12-20T16:16:54.649Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [345] : [NGSIRestHandler] NGSI event created for ContextElementResponse: {"contextElement":{"id":"Car1","type":"Car","isPattern":"false","attributes":[ {"name":"temperature","type":"centigrade","value":"26.5","metadatas":[]} ,{"name":"the_geom","type":"geometry","value":", ","metadatas":[ {"name":"location","type":"string","value":"WGS84"} ]}]},"statusCode":{"code":"200","reasonPhrase":"OK"}} cygnus-ngsi_1 | 2017-12-20T16:16:54.649781717Z time=2017-12-20T16:16:54.649Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [350] : [NGSIRestHandler] Header added to NGSI event (fiware-service: default) cygnus-ngsi_1 | 2017-12-20T16:16:54.649949133Z time=2017-12-20T16:16:54.649Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [353] : [NGSIRestHandler] Header added to NGSI event (fiware-servicepath: /) cygnus-ngsi_1 | 2017-12-20T16:16:54.650115082Z time=2017-12-20T16:16:54.649Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [356] : [NGSIRestHandler] Header added to NGSI event (fiware-correlator: 5ec1caf8-1972-4021-ac57-fb78bc48cca9) cygnus-ngsi_1 | 2017-12-20T16:16:54.650281925Z time=2017-12-20T16:16:54.650Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [359] : [NGSIRestHandler] Header added to NGSI event (transaction-id: 5ec1caf8-1972-4021-ac57-fb78bc48cca9) cygnus-ngsi_1 | 2017-12-20T16:16:54.651105909Z time=2017-12-20T16:16:54.650Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [385] : [NGSIRestHandler] NGSI events put in the channel, ids=839243882 cygnus-ngsi_1 | 2017-12-20T16:16:54.651440931Z time=2017-12-20T16:16:54.651Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor [84] : [gi] Event intercepted, id=839243882 cygnus-ngsi_1 | 2017-12-20T16:16:54.651608565Z time=2017-12-20T16:16:54.651Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor [117] : [gi] Adding flume event header (notified-entity: Car1_Car) cygnus-ngsi_1 | 2017-12-20T16:16:54.651775814Z time=2017-12-20T16:16:54.651Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor [120] : [gi] Adding flume event header (grouped-entity: Car1_Car) cygnus-ngsi_1 | 2017-12-20T16:16:54.651943558Z time=2017-12-20T16:16:54.651Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor [123] : [gi] Adding flume event header (grouped-servicepath: /) cygnus-ngsi_1 | 2017-12-20T16:16:54.652111840Z time=2017-12-20T16:16:54.651Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor [127] : [gi] Event put in the channel, id=839243882 cygnus-ngsi_1 | 2017-12-20T16:16:55.246177815Z time=2017-12-20T16:16:55.245Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=processNewBatches | msg=com.telefonica.iot.cygnus.sinks.NGSISink [554] : Batch completed cygnus-ngsi_1 | 2017-12-20T16:16:55.246220256Z time=2017-12-20T16:16:55.246Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=persistBatch | msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink [381] : [cartodb-sink] Processing sub-batch regarding the default_/_Car1_Car destination cygnus-ngsi_1 | 2017-12-20T16:16:55.257123107Z time=2017-12-20T16:16:55.256Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=aggregate | msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink$CartoDBAggregator [570] : [cartodb-sink] Processing context element (id=Car1, type=Car) cygnus-ngsi_1 | 2017-12-20T16:16:55.257152319Z time=2017-12-20T16:16:55.256Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=aggregate | msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink$CartoDBAggregator [592] : [cartodb-sink] Processing context attribute (name=temperature, type=centigrade) cygnus-ngsi_1 | 2017-12-20T16:16:55.257436049Z time=2017-12-20T16:16:55.257Z | lvl=DEBUG | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=aggregate | msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink$CartoDBAggregator [592] : [cartodb-sink] Processing context attribute (name=the_geom, type=geometry) cygnus-ngsi_1 | 2017-12-20T16:16:55.257732286Z time=2017-12-20T16:16:55.257Z | lvl=INFO | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=persistRawAggregation | msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink [616] : [cartodb-sink] Persisting data at NGSICartoDBSink. Schema (default), Table (x002fxffffx0043ar1xffffx0043ar), Data (('2017-12-20T16:16:54.651Z','/','Car1','Car',ST_SetSRID(ST_MakePoint(,), 4326),'26.5','[]')) *cygnus-ngsi_1 | 2017-12-20T16:16:55.259618557Z time=2017-12-20T16:16:55.258Z | lvl=ERROR | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=run | msg=org.apache.flume.SinkRunner$PollingRunner [160] : Unable to deliver event. Exception follows.* cygnus-ngsi_1 | 2017-12-20T16:16:55.259632910Z java.lang.NullPointerException cygnus-ngsi_1 | 2017-12-20T16:16:55.259636430Z at com.telefonica.iot.cygnus.sinks.NGSICartoDBSink.persistRawAggregation(NGSICartoDBSink.java:625) cygnus-ngsi_1 | 2017-12-20T16:16:55.259639395Z at com.telefonica.iot.cygnus.sinks.NGSICartoDBSink.persistBatch(NGSICartoDBSink.java:417) cygnus-ngsi_1 | 2017-12-20T16:16:55.259643325Z at com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:558) cygnus-ngsi_1 | 2017-12-20T16:16:55.259645857Z at com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370) cygnus-ngsi_1 | 2017-12-20T16:16:55.259648425Z at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68) cygnus-ngsi_1 | 2017-12-20T16:16:55.259651037Z at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147) cygnus-ngsi_1 | 2017-12-20T16:16:55.259653547Z at java.lang.Thread.run(Thread.java:748) cygnus-ngsi_1 | 2017-12-20T16:17:00.260728752Z time=2017-12-20T16:17:00.260Z | lvl=ERROR | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=run | msg=org.apache.flume.SinkRunner$PollingRunner [160] : Unable to deliver event. Exception follows. cygnus-ngsi_1 | 2017-12-20T16:17:00.260757599Z java.lang.IllegalStateException: begin() called when transaction is OPEN! cygnus-ngsi_1 | 2017-12-20T16:17:00.260761309Z at com.google.common.base.Preconditions.checkState(Preconditions.java:145) cygnus-ngsi_1 | 2017-12-20T16:17:00.260764359Z at org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131) cygnus-ngsi_1 | 2017-12-20T16:17:00.260767151Z at com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482) cygnus-ngsi_1 | 2017-12-20T16:17:00.260769842Z at com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370) cygnus-ngsi_1 | 2017-12-20T16:17:00.260784698Z at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68) cygnus-ngsi_1 | 2017-12-20T16:17:00.260787712Z at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147) cygnus-ngsi_1 | 2017-12-20T16:17:00.260790256Z at java.lang.Thread.run(Thread.java:748) cygnus-ngsi_1 | 2017-12-20T16:17:03.102666361Z time=2017-12-20T16:17:03.102Z | lvl=DEBUG | corr=N/A | trans=N/A | srv=N/A subsrv=N/A comp=cygnus-ngsi op=run msg=org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable [126] : Checking file:/opt/apache-flume/conf/agent.conf for changes cygnus-ngsi_1 | 2017-12-20T16:17:03.116374962Z time=2017-12-20T16:17:03.115Z | lvl=WARN | corr=N/A | trans=N/A | srv=N/A | subsrv=N/A | comp=cygnus-ngsi | op=<init> | msg=com.telefonica.iot.cygnus.interceptors.CygnusGroupingRules [49] : No grouping rules have been read. Details: /usr/cygnus/conf/grouping_rules.conf (No such file or directory) cygnus-ngsi_1 | 2017-12-20T16:17:05.261339261Z time=2017-12-20T16:17:05.260Z | lvl=ERROR | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=run | msg=org.apache.flume.SinkRunner$PollingRunner [160] : Unable to deliver event. Exception follows. cygnus-ngsi_1 | 2017-12-20T16:17:05.261368680Z java.lang.IllegalStateException: begin() called when transaction is OPEN! cygnus-ngsi_1 | 2017-12-20T16:17:05.261372739Z at com.google.common.base.Preconditions.checkState(Preconditions.java:145) cygnus-ngsi_1 | 2017-12-20T16:17:05.261376078Z at org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131) cygnus-ngsi_1 | 2017-12-20T16:17:05.261379104Z at com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482) cygnus-ngsi_1 | 2017-12-20T16:17:05.261382038Z at com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370) cygnus-ngsi_1 | 2017-12-20T16:17:05.261384947Z at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68) cygnus-ngsi_1 | 2017-12-20T16:17:05.261387928Z at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147) cygnus-ngsi_1 | 2017-12-20T16:17:05.261390817Z at java.lang.Thread.run(Thread.java:748) cygnus-ngsi_1 | 2017-12-20T16:17:10.262020846Z time=2017-12-20T16:17:10.261Z | lvl=ERROR | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=run | msg=org.apache.flume.SinkRunner$PollingRunner [160] : Unable to deliver event. Exception follows. cygnus-ngsi_1 | 2017-12-20T16:17:10.262065339Z java.lang.IllegalStateException: begin() called when transaction is OPEN! cygnus-ngsi_1 | 2017-12-20T16:17:10.262070038Z at com.google.common.base.Preconditions.checkState(Preconditions.java:145) cygnus-ngsi_1 | 2017-12-20T16:17:10.262073562Z at org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131) cygnus-ngsi_1 | 2017-12-20T16:17:10.262076685Z at com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482) cygnus-ngsi_1 | 2017-12-20T16:17:10.262079595Z at com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370) cygnus-ngsi_1 | 2017-12-20T16:17:10.262082479Z at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68) cygnus-ngsi_1 | 2017-12-20T16:17:10.262085521Z at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147) cygnus-ngsi_1 | 2017-12-20T16:17:10.262100755Z at java.lang.Thread.run(Thread.java:748) cygnus-ngsi_1 | 2017-12-20T16:17:15.262622173Z time=2017-12-20T16:17:15.261Z | lvl=ERROR | corr=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | trans=5ec1caf8-1972-4021-ac57-fb78bc48cca9 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=run | msg=org.apache.flume.SinkRunner$PollingRunner [160] : Unable to deliver event. Exception follows. On Wed, Dec 20, 2017 at 5:07 PM Help-Desk <jira-help-desk@jira.fiware.org> wrote: > >

Hide

Permalink

Andres Muñoz added a comment - 20/Dec/17 5:06 PM

If your cygnus service is listening in the port 8081 you have to run the script with that port instead of the 5050 ex:

./notification-json-geom.sh http://your-cygnus-ip:8081/notify myservice /myservicepath

Show

Andres Muñoz added a comment - 20/Dec/17 5:06 PM If your cygnus service is listening in the port 8081 you have to run the script with that port instead of the 5050 ex: ./notification-json-geom.sh http://your-cygnus-ip:8081/notify myservice /myservicepath

Hide

Permalink

FW External User added a comment - 20/Dec/17 4:24 PM

Comment by peter@familiohq.com :

Yes, great – got it. When I run the script I get connection refused ..
like I can't connect to the "notify" endpoint.

The cygnus API is running fine, though i.e. if I do

curl "http://api.familiohq.com:8081/v1/version"

I get

{ "success": "true", "version": "1.8.0_SNAPSHOT.3f677284750cd601756a67a72a20267090efdff8" }

On Wed, Dec 20, 2017 at 3:48 PM Help-Desk <jira-help-desk@jira.fiware.org>
wrote:

>
>

Show

FW External User added a comment - 20/Dec/17 4:24 PM Comment by peter@familiohq.com : Yes, great – got it. When I run the script I get connection refused .. like I can't connect to the "notify" endpoint. The cygnus API is running fine, though i.e. if I do curl "http://api.familiohq.com:8081/v1/version" I get { "success": "true", "version": "1.8.0_SNAPSHOT.3f677284750cd601756a67a72a20267090efdff8" } On Wed, Dec 20, 2017 at 3:48 PM Help-Desk <jira-help-desk@jira.fiware.org> wrote: > >

Hide

Permalink

Andres Muñoz added a comment - 20/Dec/17 3:47 PM

notification-json-geom.sh

Hi,

I sent to your personal email and to this comment

Show

Andres Muñoz added a comment - 20/Dec/17 3:47 PM notification-json-geom.sh Hi, I sent to your personal email and to this comment

Hide

Permalink

FW External User added a comment - 20/Dec/17 3:40 PM

Comment by peter@familiohq.com :

Hi Andres,

I haven't received the script file .. it's not attached to the emails I'm
getting. Are you able to send it directly to my email at peter@familiohq.com –
thanks

On Wed, Dec 20, 2017 at 3:27 PM Help-Desk <jira-help-desk@jira.fiware.org>
wrote:

>
>

Show

FW External User added a comment - 20/Dec/17 3:40 PM Comment by peter@familiohq.com : Hi Andres, I haven't received the script file .. it's not attached to the emails I'm getting. Are you able to send it directly to my email at peter@familiohq.com – thanks On Wed, Dec 20, 2017 at 3:27 PM Help-Desk <jira-help-desk@jira.fiware.org> wrote: > >

Hide

Permalink

Andres Muñoz added a comment - 20/Dec/17 3:26 PM

Hello,

Did you make the test that I wrote you before?

Show

Andres Muñoz added a comment - 20/Dec/17 3:26 PM Hello, Did you make the test that I wrote you before?

Hide

Permalink

FW External User added a comment - 20/Dec/17 1:49 PM

Comment by peter@familiohq.com :

Hi Andres,

Anything news on this? Also are you able to provide me with your
"agent.conf" and "cartodb_keys.conf" file (removing your actual key is
fine) so I'm able to compare it with mine to see if there is anything
different? Maybe also your "docker-compose.yml" file if you're using Docker
to setup orion/cygnus.

Thanks!

On Mon, Dec 11, 2017 at 4:07 PM Help-Desk <jira-help-desk@jira.fiware.org>
wrote:

>
>

Show

FW External User added a comment - 20/Dec/17 1:49 PM Comment by peter@familiohq.com : Hi Andres, Anything news on this? Also are you able to provide me with your "agent.conf" and "cartodb_keys.conf" file (removing your actual key is fine) so I'm able to compare it with mine to see if there is anything different? Maybe also your "docker-compose.yml" file if you're using Docker to setup orion/cygnus. Thanks! On Mon, Dec 11, 2017 at 4:07 PM Help-Desk <jira-help-desk@jira.fiware.org> wrote: > >

Hide

Permalink

Andres Muñoz added a comment - 12/Dec/17 12:05 PM - edited

Hello,

I have been checking your agent.conf file and It looks good configurated, the log error (null pointer exception) that you sent us give me some clues about the creation of the object to insert to de cartodb (I am not sure, I am speculating), so I want to be sure that cygnus can handle and convert the event to the object, to this comment I am attaching a script that generates a test event and sen this to cygnus in order to persist in the cartoDB the event contains geolocation data so is pretty similar to the events that you generate. please execute this script as follows:

./notification-json-geom.sh http://your-cygnus-ip:5050/notify myservice /myservicepath

and let me know if it event is persisted in cartodb or send me again the cygnus log

Show

Andres Muñoz added a comment - 12/Dec/17 12:05 PM - edited Hello, I have been checking your agent.conf file and It looks good configurated, the log error (null pointer exception) that you sent us give me some clues about the creation of the object to insert to de cartodb (I am not sure, I am speculating), so I want to be sure that cygnus can handle and convert the event to the object, to this comment I am attaching a script that generates a test event and sen this to cygnus in order to persist in the cartoDB the event contains geolocation data so is pretty similar to the events that you generate. please execute this script as follows: ./notification-json-geom.sh http://your-cygnus-ip:5050/notify myservice /myservicepath and let me know if it event is persisted in cartodb or send me again the cygnus log

Hide

Permalink

Andres Muñoz added a comment - 11/Dec/17 4:06 PM

Hello,

I am trying to replicate the error with the settings of your agent.conf file, please give a little more of time.

Show

Andres Muñoz added a comment - 11/Dec/17 4:06 PM Hello, I am trying to replicate the error with the settings of your agent.conf file, please give a little more of time.

Hide

Permalink

FW External User added a comment - 11/Dec/17 10:51 AM

Comment by peter@familiohq.com :

Hi guys,

Have you been able to look at my issue? I would appreciate your help with
this issue.

On Mon, Dec 4, 2017 at 1:53 PM Peter Theill <peter@familiohq.com> wrote:

> Sure, this is my file agent.conf
>
> cygnus-ngsi.sources = http-source
>
> cygnus-ngsi.sinks = cartodb-sink
>
> cygnus-ngsi.channels = cartodb-channel
>
>
> cygnus-ngsi.sources.http-source.channels = cartodb-channel
>
> cygnus-ngsi.sources.http-source.type =
> org.apache.flume.source.http.HTTPSource
>
> cygnus-ngsi.sources.http-source.port = 5050
>
> cygnus-ngsi.sources.http-source.handler =
> com.telefonica.iot.cygnus.handlers.NGSIRestHandler
>
> cygnus-ngsi.sources.http-source.handler.notification_target = /notify
>
> cygnus-ngsi.sources.http-source.handler.default_service = default
>
> cygnus-ngsi.sources.http-source.handler.default_service_path = /
>
> cygnus-ngsi.sources.http-source.interceptors = ts gi
>
> cygnus-ngsi.sources.http-source.interceptors.ts.type = timestamp
>
> cygnus-ngsi.sources.http-source.interceptors.gi.type =
> com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor$Builder
>
> cygnus-ngsi.sources.http-source.interceptors.gi.grouping_rules_conf_file =
> /usr/cygnus/conf/grouping_rules.conf
>
>
> cygnus-ngsi.sinks.cartodb-sink.type =
> com.telefonica.iot.cygnus.sinks.NGSICartoDBSink
>
> cygnus-ngsi.sinks.cartodb-sink.channel = cartodb-channel
>
> cygnus-ngsi.sinks.cartodb-sink.keys_conf_file =
> /opt/apache-flume/conf/cartodb_keys.conf
>
>
> cygnus-ngsi.channels.cartodb-channel.type = memory
>
> cygnus-ngsi.channels.cartodb-channel.capacity = 1000
>
> cygnus-ngsi.channels.cartodb-channel.transactionCapacity = 100
>
>
> Let me know if you want more info.
>
> On Mon, Dec 4, 2017 at 1:13 PM Help-Desk <jira-help-desk@jira.fiware.org>
> wrote:
>
>>
>>
>

Show

FW External User added a comment - 11/Dec/17 10:51 AM Comment by peter@familiohq.com : Hi guys, Have you been able to look at my issue? I would appreciate your help with this issue. On Mon, Dec 4, 2017 at 1:53 PM Peter Theill <peter@familiohq.com> wrote: > Sure, this is my file agent.conf > > cygnus-ngsi.sources = http-source > > cygnus-ngsi.sinks = cartodb-sink > > cygnus-ngsi.channels = cartodb-channel > > > cygnus-ngsi.sources.http-source.channels = cartodb-channel > > cygnus-ngsi.sources.http-source.type = > org.apache.flume.source.http.HTTPSource > > cygnus-ngsi.sources.http-source.port = 5050 > > cygnus-ngsi.sources.http-source.handler = > com.telefonica.iot.cygnus.handlers.NGSIRestHandler > > cygnus-ngsi.sources.http-source.handler.notification_target = /notify > > cygnus-ngsi.sources.http-source.handler.default_service = default > > cygnus-ngsi.sources.http-source.handler.default_service_path = / > > cygnus-ngsi.sources.http-source.interceptors = ts gi > > cygnus-ngsi.sources.http-source.interceptors.ts.type = timestamp > > cygnus-ngsi.sources.http-source.interceptors.gi.type = > com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor$Builder > > cygnus-ngsi.sources.http-source.interceptors.gi.grouping_rules_conf_file = > /usr/cygnus/conf/grouping_rules.conf > > > cygnus-ngsi.sinks.cartodb-sink.type = > com.telefonica.iot.cygnus.sinks.NGSICartoDBSink > > cygnus-ngsi.sinks.cartodb-sink.channel = cartodb-channel > > cygnus-ngsi.sinks.cartodb-sink.keys_conf_file = > /opt/apache-flume/conf/cartodb_keys.conf > > > cygnus-ngsi.channels.cartodb-channel.type = memory > > cygnus-ngsi.channels.cartodb-channel.capacity = 1000 > > cygnus-ngsi.channels.cartodb-channel.transactionCapacity = 100 > > > Let me know if you want more info. > > On Mon, Dec 4, 2017 at 1:13 PM Help-Desk <jira-help-desk@jira.fiware.org> > wrote: > >> >> >

Hide

Permalink

FW External User added a comment - 04/Dec/17 1:54 PM

Comment by peter@familiohq.com :

Sure, this is my file agent.conf

cygnus-ngsi.sources = http-source

cygnus-ngsi.sinks = cartodb-sink

cygnus-ngsi.channels = cartodb-channel

cygnus-ngsi.sources.http-source.channels = cartodb-channel

cygnus-ngsi.sources.http-source.type =
org.apache.flume.source.http.HTTPSource

cygnus-ngsi.sources.http-source.port = 5050

cygnus-ngsi.sources.http-source.handler =
com.telefonica.iot.cygnus.handlers.NGSIRestHandler

cygnus-ngsi.sources.http-source.handler.notification_target = /notify

cygnus-ngsi.sources.http-source.handler.default_service = default

cygnus-ngsi.sources.http-source.handler.default_service_path = /

cygnus-ngsi.sources.http-source.interceptors = ts gi

cygnus-ngsi.sources.http-source.interceptors.ts.type = timestamp

cygnus-ngsi.sources.http-source.interceptors.gi.type =
com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor$Builder

cygnus-ngsi.sources.http-source.interceptors.gi.grouping_rules_conf_file =
/usr/cygnus/conf/grouping_rules.conf

cygnus-ngsi.sinks.cartodb-sink.type =
com.telefonica.iot.cygnus.sinks.NGSICartoDBSink

cygnus-ngsi.sinks.cartodb-sink.channel = cartodb-channel

cygnus-ngsi.sinks.cartodb-sink.keys_conf_file =
/opt/apache-flume/conf/cartodb_keys.conf

cygnus-ngsi.channels.cartodb-channel.type = memory

cygnus-ngsi.channels.cartodb-channel.capacity = 1000

cygnus-ngsi.channels.cartodb-channel.transactionCapacity = 100

Let me know if you want more info.

On Mon, Dec 4, 2017 at 1:13 PM Help-Desk <jira-help-desk@jira.fiware.org>
wrote:

>
>

Show

FW External User added a comment - 04/Dec/17 1:54 PM Comment by peter@familiohq.com : Sure, this is my file agent.conf cygnus-ngsi.sources = http-source cygnus-ngsi.sinks = cartodb-sink cygnus-ngsi.channels = cartodb-channel cygnus-ngsi.sources.http-source.channels = cartodb-channel cygnus-ngsi.sources.http-source.type = org.apache.flume.source.http.HTTPSource cygnus-ngsi.sources.http-source.port = 5050 cygnus-ngsi.sources.http-source.handler = com.telefonica.iot.cygnus.handlers.NGSIRestHandler cygnus-ngsi.sources.http-source.handler.notification_target = /notify cygnus-ngsi.sources.http-source.handler.default_service = default cygnus-ngsi.sources.http-source.handler.default_service_path = / cygnus-ngsi.sources.http-source.interceptors = ts gi cygnus-ngsi.sources.http-source.interceptors.ts.type = timestamp cygnus-ngsi.sources.http-source.interceptors.gi.type = com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor$Builder cygnus-ngsi.sources.http-source.interceptors.gi.grouping_rules_conf_file = /usr/cygnus/conf/grouping_rules.conf cygnus-ngsi.sinks.cartodb-sink.type = com.telefonica.iot.cygnus.sinks.NGSICartoDBSink cygnus-ngsi.sinks.cartodb-sink.channel = cartodb-channel cygnus-ngsi.sinks.cartodb-sink.keys_conf_file = /opt/apache-flume/conf/cartodb_keys.conf cygnus-ngsi.channels.cartodb-channel.type = memory cygnus-ngsi.channels.cartodb-channel.capacity = 1000 cygnus-ngsi.channels.cartodb-channel.transactionCapacity = 100 Let me know if you want more info. On Mon, Dec 4, 2017 at 1:13 PM Help-Desk <jira-help-desk@jira.fiware.org> wrote: > >

Hide

Permalink

Joaquin Salvachua added a comment - 04/Dec/17 1:12 PM

Hello,

I am checking the issue with your development, but I need a little more of information, for trying to replicate the error, Can you send me your cygnus agent configuration file?

Show

Joaquin Salvachua added a comment - 04/Dec/17 1:12 PM Hello, I am checking the issue with your development, but I need a little more of information, for trying to replicate the error, Can you send me your cygnus agent configuration file?

Hide

Permalink

FW External User added a comment - 27/Nov/17 5:58 AM

Comment by peter@familiohq.com :

Dear FIWARE,

Did you have a chance to look into this issue further? I'm a bit blocked in
my further development.

On Thu, Nov 23, 2017 at 3:02 PM Help-Desk <jira-help-desk@jira.fiware.org>
wrote:

>
>

Show

FW External User added a comment - 27/Nov/17 5:58 AM Comment by peter@familiohq.com : Dear FIWARE, Did you have a chance to look into this issue further? I'm a bit blocked in my further development. On Thu, Nov 23, 2017 at 3:02 PM Help-Desk <jira-help-desk@jira.fiware.org> wrote: > >

Hide

Permalink

FW External User added a comment - 23/Nov/17 11:09 AM

Comment by jsalvachua@dit.upm.es :

Dear Peter,

we will have a look at it. Perhaps we may ask you a bit about your configuration.

Best Regards

Joaquín

> El 23 nov 2017, a las 14:53, Peter Theill <peter@familiohq.com> escribió:
>
> I've setup Orion and Cygnus so whenever a new entity is created in Orion it's transferring its data to Carto using the Cygnus CartoDbSink.
>
> However I'm having some issues with this implementation. I'm getting the data correctly from Orion to Cygnus but when the sink tries to forward those data to Carto I'm getting this error message (also attached for better formatting) when trying to persist it in Carto
>
> cygnus_1 | time=2017-11-23T13:40:11.175Z | lvl=INFO | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[294] : [NGSIRestHandler] Starting internal transaction (efab5d17-12c3-4716-8b1b-15352ec6ae18)
>
> cygnus_1 | time=2017-11-23T13:40:11.176Z | lvl=INFO | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[312] : [NGSIRestHandler] Received data ({"subscriptionId":"5a16cd57fde41d458740a0b4","originator":"localhost","contextResponses":[{"contextElement":{"type":"Event","isPattern":"false","id":"getif","attributes":[

{"name":"location","type":"geo:point","value":"79.00555, -148.99808"}

]},"statusCode":{"code":"200","reasonPhrase":"OK"}}]})
>
> cygnus_1 | time=2017-11-23T13:40:11.239Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[320] : [NGSIRestHandler] Parsed NotifyContextRequest: {"subscriptionId":"5a16cd57fde41d458740a0b4","originator":"localhost","contextResponses":[{"contextElement":{"id":"getif","type":"Event","isPattern":"false","attributes":[

{"name":"location","type":"geo:point","value":"79.00555, -148.99808","metadatas":[]}

]},"statusCode":{"code":"200","reasonPhrase":"OK"}}]}
>
> cygnus_1 | time=2017-11-23T13:40:11.239Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[345] : [NGSIRestHandler] NGSI event created for ContextElementResponse: {"contextElement":{"id":"getif","type":"Event","isPattern":"false","attributes":[

{"name":"location","type":"geo:point","value":"79.00555, -148.99808","metadatas":[]}

]},"statusCode":{"code":"200","reasonPhrase":"OK"}}
>
> cygnus_1 | time=2017-11-23T13:40:11.239Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[350] : [NGSIRestHandler] Header added to NGSI event (fiware-service: default)
>
> cygnus_1 | time=2017-11-23T13:40:11.239Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[353] : [NGSIRestHandler] Header added to NGSI event (fiware-servicepath: /)
>
> cygnus_1 | time=2017-11-23T13:40:11.239Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[356] : [NGSIRestHandler] Header added to NGSI event (fiware-correlator: d3be117c-d053-11e7-ad11-0242ac120009)
>
> cygnus_1 | time=2017-11-23T13:40:11.240Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[359] : [NGSIRestHandler] Header added to NGSI event (transaction-id: efab5d17-12c3-4716-8b1b-15352ec6ae18)
>
> cygnus_1 | time=2017-11-23T13:40:11.240Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler[385] : [NGSIRestHandler] NGSI events put in the channel, ids=1516398619
>
> cygnus_1 | time=2017-11-23T13:40:11.241Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor[84] : [gi] Event intercepted, id=1516398619
>
> cygnus_1 | time=2017-11-23T13:40:11.241Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor[117] : [gi] Adding flume event header (notified-entity: getif_Event)
>
> cygnus_1 | time=2017-11-23T13:40:11.241Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor[120] : [gi] Adding flume event header (grouped-entity: getif_Event)
>
> cygnus_1 | time=2017-11-23T13:40:11.241Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor[123] : [gi] Adding flume event header (grouped-servicepath: /)
>
> cygnus_1 | time=2017-11-23T13:40:11.241Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor[127] : [gi] Event put in the channel, id=1516398619
>
> orion_1 | time=Thursday 23 Nov 13:40:11 2017.243Z | lvl=INFO | corr=N/A | trans=1511444367-204-00000000002 | from=pending | srv=pending | subsrv=pending | comp=Orion | op=httpRequestSend.cpp[620]:httpRequestSendWithCurl | msg=Notification Successfully Sent to http://staging.familiohq.com:5050/notify <http://staging.familiohq.com:5050/notify>
> orion_1 | time=Thursday 23 Nov 13:40:11 2017.243Z | lvl=INFO | corr=N/A | trans=1511444367-204-00000000002 | from=pending | srv=pending | subsrv=pending | comp=Orion | op=logMsg.h[1916]:lmTransactionEnd | msg=Transaction ended
>
> cygnus_1 | time=2017-11-23T13:40:11.244Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=processNewBatches | msg=com.telefonica.iot.cygnus.sinks.NGSISink[554] : Batch completed
>
> cygnus_1 | time=2017-11-23T13:40:11.244Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=persistBatch | msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink[381] : [cartodb-sink] Processing sub-batch regarding the default_/_getif_Event destination
>
> cygnus_1 | time=2017-11-23T13:40:11.254Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=aggregate | msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink$CartoDBAggregator[570] : [cartodb-sink] Processing context element (id=getif, type=Event)
>
> cygnus_1 | time=2017-11-23T13:40:11.254Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=aggregate | msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink$CartoDBAggregator[592] : [cartodb-sink] Processing context attribute (name=location, type=geo:point)
>
> cygnus_1 | time=2017-11-23T13:40:11.254Z | lvl=INFO | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=persistRawAggregation | msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink[616] : [cartodb-sink] Persisting data at NGSICartoDBSink. Schema (default), Table (x002fxffffgetifxffffx0045vent), Data (('2017-11-23T13:40:11.241Z','/','getif','Event',ST_SetSRID(ST_MakePoint(79.00555,-148.99808), 4326)))
>
> cygnus_1 | time=2017-11-23T13:40:11.255Z | lvl=ERROR | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=run | msg=org.apache.flume.SinkRunner$PollingRunner[160] : Unable to deliver event. Exception follows.
>
> cygnus_1 | java.lang.NullPointerException
>
> cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSICartoDBSink.persistRawAggregation(NGSICartoDBSink.java:625)
>
> cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSICartoDBSink.persistBatch(NGSICartoDBSink.java:417)
>
> cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:558)
>
> cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370)
>
> cygnus_1 | at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)
>
> cygnus_1 | at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)
>
> cygnus_1 | at java.lang.Thread.run(Thread.java:748)
>
> cygnus_1 | time=2017-11-23T13:40:16.256Z | lvl=ERROR | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=run | msg=org.apache.flume.SinkRunner$PollingRunner[160] : Unable to deliver event. Exception follows.
>
> cygnus_1 | java.lang.IllegalStateException: begin() called when transaction is OPEN!
>
> cygnus_1 | at com.google.common.base.Preconditions.checkState(Preconditions.java:145)
>
> cygnus_1 | at org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131)
>
> cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482)
>
> cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370)
>
> cygnus_1 | at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)
>
> cygnus_1 | at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)
>
> cygnus_1 | at java.lang.Thread.run(Thread.java:748)
>
> cygnus_1 | time=2017-11-23T13:40:18.648Z | lvl=DEBUG | corr=N/A | trans=N/A | srv=N/A | subsrv=N/A | comp=cygnus-ngsi | op=run | msg=org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable[126] : Checking file:/opt/apache-flume/conf/agent.conf for changes
>
> cygnus_1 | time=2017-11-23T13:40:18.654Z | lvl=WARN | corr=N/A | trans=N/A | srv=N/A | subsrv=N/A | comp=cygnus-ngsi | op=<init> | msg=com.telefonica.iot.cygnus.interceptors.CygnusGroupingRules[49] : No grouping rules have been read. Details: /usr/cygnus/conf/grouping_rules.conf (No such file or directory)
>
> cygnus_1 | time=2017-11-23T13:40:21.257Z | lvl=ERROR | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=run | msg=org.apache.flume.SinkRunner$PollingRunner[160] : Unable to deliver event. Exception follows.
>
> cygnus_1 | java.lang.IllegalStateException: begin() called when transaction is OPEN!
>
> cygnus_1 | at com.google.common.base.Preconditions.checkState(Preconditions.java:145)
>
> cygnus_1 | at org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131)
>
> cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482)
>
> cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370)
>
> cygnus_1 | at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)
>
> cygnus_1 | at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)
>
> cygnus_1 | at java.lang.Thread.run(Thread.java:748)
>
> cygnus_1 | time=2017-11-23T13:40:26.258Z | lvl=ERROR | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=run | msg=org.apache.flume.SinkRunner$PollingRunner[160] : Unable to deliver event. Exception follows.
>
> cygnus_1 | java.lang.IllegalStateException: begin() called when transaction is OPEN!
>
> cygnus_1 | at com.google.common.base.Preconditions.checkState(Preconditions.java:145)
>
> cygnus_1 | at org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131)
>
> cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482)
>
> cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370)
>
> cygnus_1 | at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)
>
> cygnus_1 | at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)
>
> cygnus_1 | at java.lang.Thread.run(Thread.java:748)
>
>
>
> Are you able to help me figure out why I can't send this information to carto?
>
>
>
> –
> Peter Theill
> CTO
> +45 6171 5096 <tel:61%2071%2050%2096>
> peter@familiohq.com <peter@familiohq.com>
>
> Familio
> Artillerivej 86
> DK-2300 Copenhagen S
> https://familiohq.com/ <https://familiohq.com/><carto-cygnus-log.txt>

Show

FW External User added a comment - 23/Nov/17 11:09 AM Comment by jsalvachua@dit.upm.es : Dear Peter, we will have a look at it. Perhaps we may ask you a bit about your configuration. Best Regards Joaquín > El 23 nov 2017, a las 14:53, Peter Theill <peter@familiohq.com> escribió: > > I've setup Orion and Cygnus so whenever a new entity is created in Orion it's transferring its data to Carto using the Cygnus CartoDbSink. > > However I'm having some issues with this implementation. I'm getting the data correctly from Orion to Cygnus but when the sink tries to forward those data to Carto I'm getting this error message (also attached for better formatting) when trying to persist it in Carto > > cygnus_1 | time=2017-11-23T13:40:11.175Z | lvl=INFO | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [294] : [NGSIRestHandler] Starting internal transaction (efab5d17-12c3-4716-8b1b-15352ec6ae18) > > cygnus_1 | time=2017-11-23T13:40:11.176Z | lvl=INFO | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [312] : [NGSIRestHandler] Received data ({"subscriptionId":"5a16cd57fde41d458740a0b4","originator":"localhost","contextResponses":[{"contextElement":{"type":"Event","isPattern":"false","id":"getif","attributes":[ {"name":"location","type":"geo:point","value":"79.00555, -148.99808"} ]},"statusCode":{"code":"200","reasonPhrase":"OK"}}]}) > > cygnus_1 | time=2017-11-23T13:40:11.239Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [320] : [NGSIRestHandler] Parsed NotifyContextRequest: {"subscriptionId":"5a16cd57fde41d458740a0b4","originator":"localhost","contextResponses":[{"contextElement":{"id":"getif","type":"Event","isPattern":"false","attributes":[ {"name":"location","type":"geo:point","value":"79.00555, -148.99808","metadatas":[]} ]},"statusCode":{"code":"200","reasonPhrase":"OK"}}]} > > cygnus_1 | time=2017-11-23T13:40:11.239Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [345] : [NGSIRestHandler] NGSI event created for ContextElementResponse: {"contextElement":{"id":"getif","type":"Event","isPattern":"false","attributes":[ {"name":"location","type":"geo:point","value":"79.00555, -148.99808","metadatas":[]} ]},"statusCode":{"code":"200","reasonPhrase":"OK"}} > > cygnus_1 | time=2017-11-23T13:40:11.239Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [350] : [NGSIRestHandler] Header added to NGSI event (fiware-service: default) > > cygnus_1 | time=2017-11-23T13:40:11.239Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [353] : [NGSIRestHandler] Header added to NGSI event (fiware-servicepath: /) > > cygnus_1 | time=2017-11-23T13:40:11.239Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [356] : [NGSIRestHandler] Header added to NGSI event (fiware-correlator: d3be117c-d053-11e7-ad11-0242ac120009) > > cygnus_1 | time=2017-11-23T13:40:11.240Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [359] : [NGSIRestHandler] Header added to NGSI event (transaction-id: efab5d17-12c3-4716-8b1b-15352ec6ae18) > > cygnus_1 | time=2017-11-23T13:40:11.240Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=getEvents | msg=com.telefonica.iot.cygnus.handlers.NGSIRestHandler [385] : [NGSIRestHandler] NGSI events put in the channel, ids=1516398619 > > cygnus_1 | time=2017-11-23T13:40:11.241Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor [84] : [gi] Event intercepted, id=1516398619 > > cygnus_1 | time=2017-11-23T13:40:11.241Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor [117] : [gi] Adding flume event header (notified-entity: getif_Event) > > cygnus_1 | time=2017-11-23T13:40:11.241Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor [120] : [gi] Adding flume event header (grouped-entity: getif_Event) > > cygnus_1 | time=2017-11-23T13:40:11.241Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor [123] : [gi] Adding flume event header (grouped-servicepath: /) > > cygnus_1 | time=2017-11-23T13:40:11.241Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=intercept | msg=com.telefonica.iot.cygnus.interceptors.NGSIGroupingInterceptor [127] : [gi] Event put in the channel, id=1516398619 > > orion_1 | time=Thursday 23 Nov 13:40:11 2017.243Z | lvl=INFO | corr=N/A | trans=1511444367-204-00000000002 | from=pending | srv=pending | subsrv=pending | comp=Orion | op=httpRequestSend.cpp [620] :httpRequestSendWithCurl | msg=Notification Successfully Sent to http://staging.familiohq.com:5050/notify < http://staging.familiohq.com:5050/notify > > orion_1 | time=Thursday 23 Nov 13:40:11 2017.243Z | lvl=INFO | corr=N/A | trans=1511444367-204-00000000002 | from=pending | srv=pending | subsrv=pending | comp=Orion | op=logMsg.h [1916] :lmTransactionEnd | msg=Transaction ended > > cygnus_1 | time=2017-11-23T13:40:11.244Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=processNewBatches | msg=com.telefonica.iot.cygnus.sinks.NGSISink [554] : Batch completed > > cygnus_1 | time=2017-11-23T13:40:11.244Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=persistBatch | msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink [381] : [cartodb-sink] Processing sub-batch regarding the default_/_getif_Event destination > > cygnus_1 | time=2017-11-23T13:40:11.254Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=aggregate | msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink$CartoDBAggregator [570] : [cartodb-sink] Processing context element (id=getif, type=Event) > > cygnus_1 | time=2017-11-23T13:40:11.254Z | lvl=DEBUG | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=aggregate | msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink$CartoDBAggregator [592] : [cartodb-sink] Processing context attribute (name=location, type=geo:point) > > cygnus_1 | time=2017-11-23T13:40:11.254Z | lvl=INFO | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=persistRawAggregation | msg=com.telefonica.iot.cygnus.sinks.NGSICartoDBSink [616] : [cartodb-sink] Persisting data at NGSICartoDBSink. Schema (default), Table (x002fxffffgetifxffffx0045vent), Data (('2017-11-23T13:40:11.241Z','/','getif','Event',ST_SetSRID(ST_MakePoint(79.00555,-148.99808), 4326))) > > cygnus_1 | time=2017-11-23T13:40:11.255Z | lvl=ERROR | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=run | msg=org.apache.flume.SinkRunner$PollingRunner [160] : Unable to deliver event. Exception follows. > > cygnus_1 | java.lang.NullPointerException > > cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSICartoDBSink.persistRawAggregation(NGSICartoDBSink.java:625) > > cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSICartoDBSink.persistBatch(NGSICartoDBSink.java:417) > > cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:558) > > cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370) > > cygnus_1 | at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68) > > cygnus_1 | at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147) > > cygnus_1 | at java.lang.Thread.run(Thread.java:748) > > cygnus_1 | time=2017-11-23T13:40:16.256Z | lvl=ERROR | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=run | msg=org.apache.flume.SinkRunner$PollingRunner [160] : Unable to deliver event. Exception follows. > > cygnus_1 | java.lang.IllegalStateException: begin() called when transaction is OPEN! > > cygnus_1 | at com.google.common.base.Preconditions.checkState(Preconditions.java:145) > > cygnus_1 | at org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131) > > cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482) > > cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370) > > cygnus_1 | at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68) > > cygnus_1 | at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147) > > cygnus_1 | at java.lang.Thread.run(Thread.java:748) > > cygnus_1 | time=2017-11-23T13:40:18.648Z | lvl=DEBUG | corr=N/A | trans=N/A | srv=N/A | subsrv=N/A | comp=cygnus-ngsi | op=run | msg=org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable [126] : Checking file:/opt/apache-flume/conf/agent.conf for changes > > cygnus_1 | time=2017-11-23T13:40:18.654Z | lvl=WARN | corr=N/A | trans=N/A | srv=N/A | subsrv=N/A | comp=cygnus-ngsi | op=<init> | msg=com.telefonica.iot.cygnus.interceptors.CygnusGroupingRules [49] : No grouping rules have been read. Details: /usr/cygnus/conf/grouping_rules.conf (No such file or directory) > > cygnus_1 | time=2017-11-23T13:40:21.257Z | lvl=ERROR | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=run | msg=org.apache.flume.SinkRunner$PollingRunner [160] : Unable to deliver event. Exception follows. > > cygnus_1 | java.lang.IllegalStateException: begin() called when transaction is OPEN! > > cygnus_1 | at com.google.common.base.Preconditions.checkState(Preconditions.java:145) > > cygnus_1 | at org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131) > > cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482) > > cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370) > > cygnus_1 | at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68) > > cygnus_1 | at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147) > > cygnus_1 | at java.lang.Thread.run(Thread.java:748) > > cygnus_1 | time=2017-11-23T13:40:26.258Z | lvl=ERROR | corr=d3be117c-d053-11e7-ad11-0242ac120009 | trans=efab5d17-12c3-4716-8b1b-15352ec6ae18 | srv=default | subsrv=/ | comp=cygnus-ngsi | op=run | msg=org.apache.flume.SinkRunner$PollingRunner [160] : Unable to deliver event. Exception follows. > > cygnus_1 | java.lang.IllegalStateException: begin() called when transaction is OPEN! > > cygnus_1 | at com.google.common.base.Preconditions.checkState(Preconditions.java:145) > > cygnus_1 | at org.apache.flume.channel.BasicTransactionSemantics.begin(BasicTransactionSemantics.java:131) > > cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.processNewBatches(NGSISink.java:482) > > cygnus_1 | at com.telefonica.iot.cygnus.sinks.NGSISink.process(NGSISink.java:370) > > cygnus_1 | at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68) > > cygnus_1 | at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147) > > cygnus_1 | at java.lang.Thread.run(Thread.java:748) > > > > Are you able to help me figure out why I can't send this information to carto? > > > > – > Peter Theill > CTO > +45 6171 5096 <tel:61%2071%2050%2096> > peter@familiohq.com < peter@familiohq.com > > > Familio > Artillerivej 86 > DK-2300 Copenhagen S > https://familiohq.com/ < https://familiohq.com/ ><carto-cygnus-log.txt>

People

Assignee:

Andres Muñoz

Reporter:

FW External User

Votes:

0 Vote for this issue

Watchers:

4 Start watching this issue

Dates

Created:

23/Nov/17 11:02 AM

Updated:

27/May/21 10:57 AM

Resolved:

06/Feb/18 10:10 AM

Agile

View on Board