GlassBox
Published in

GlassBox

Creating a Twitter Graph Using Slash GraphQL

The Approach

Retrieving Data From Twitter Using `twarc`

  • CONSUMER_KEY
  • CONSUMER_SECRET
  • ACCESS_TOKEN
  • ACCESS_TOKEN_SECRET
twarc search #NeilPeart > tweets.jsonl
{
"content": "It’s been one year since he passed, but the music lives on... he also probably wouldn’t have liked what was going on in the word. Keep resting easy, Neil #NeilPeart https://t.co/pTidwTYsxG",
"tweet_id": "1347203390560940035",
"user": {
"screen_name": "Austin Miller",
"handle": "AMiller1397"
}
}

Preparing Dgraph Slash GraphQL

type User {
handle: String! @id @search(by: [hash])
screen_name: String! @search(by: [fulltext])
}
type Tweet {
tweet_id: String! @id @search(by: [hash])
content: String!
user: User
}
type Configuration {
id: ID
search_string: String!
}

Loading Data into Slash GraphQL Using Python

  • convert — processes the JSON data to identify any Twitter mentions to another user
  • upload — prepares and performs the upload of JSON data into Slash GraphQL
data = {}
users = {}
gather_tweets_by_user()search_string = os.getenv('TWARC_SEARCH_STRING')
print(search_string)
upload_to_slash(create_configuration_query(search_string))for handle in data:
print("=====")
upload_to_slash(create_add_tweets_query(users[handle], data[handle]))
  1. The gather_tweets_by_user()organizes the Twitter data into the data and users objects.
  2. The upload_to_lash(create_configuration_query(search_string)) stores the search that was performed into Slash GraphQL for use by the Angular client
  3. The for loop processes the data and user objects, uploading each record into Slash GraphQL using upload_to_slash(create_add_tweets_query(users[handle], data[handle]))
query MyQuery {
queryTweet {
content
tweet_id
user {
screen_name
handle
}
}
}
query MyQuery {
queryConfiguration {
search_string
}
}

Using `ngx-graph` With Angular CLI

npm install @swimlane/ngx-graph --save
@NgModule({
declarations: [
AppComponent
],
imports: [
BrowserModule,
BrowserAnimationsModule,
AppRoutingModule,
HttpClientModule,
NgxGraphModule,
NgxChartsModule,
NgxSpinnerModule
],
schemas: [CUSTOM_ELEMENTS_SCHEMA],
providers: [],
bootstrap: [AppComponent]
})
export class AppModule { }
allTweets:string = 'query MyQuery {' +
' queryTweet {' +
' content' +
' tweet_id' +
' user {' +
' screen_name' +
' handle' +
' }' +
' }' +
'}';
getTweetData() {
return this.http.get<QueryResult>(this.baseUrl + '?query=' + this.allTweets).pipe(
tap(),
catchError(err => { return ErrorUtils.errorHandler(err)
}));
}

Preparing Slash GraphQL to Work With `ngx-graph`

createGraphPayload(queryResult: QueryResult): GraphPayload {
let graphPayload: GraphPayload = new GraphPayload();
if (queryResult) {
if (queryResult.data && queryResult.data.queryTweet) {
let tweetList: QueryTweet[] = queryResult.data.queryTweet;
tweetList.forEach( (queryTweet) => {
let tweetNode:GraphNode = this.getTweetNode(queryTweet, graphPayload);
let userNode:GraphNode = this.getUserNode(queryTweet, graphPayload);
if (tweetNode && userNode) {
let graphEdge:GraphEdge = new GraphEdge();
graphEdge.id = ConversionService.createRandomId();
if (tweetNode.label.substring(0, 2) === 'RT') {
graphEdge.label = 'retweet';
} else {
graphEdge.label = 'tweet';
}
graphEdge.source = userNode.id;
graphEdge.target = tweetNode.id;
graphPayload.links.push(graphEdge);
}
});
}
}
console.log('graphPayload', graphPayload);
return graphPayload;
}
export class GraphPayload {
links: GraphEdge[] = [];
nodes: GraphNode[] = [];
}
export class GraphEdge implements Edge {
id: string;
label: string;
source: string;
target: string;
}
export class GraphNode implements Node {
id: string;
label: string;
twitter_uri: string;
}

Configuring the Angular View

ngOnInit() {
this.spinner.show();
this.graphQlService.getConfigurationData().subscribe(configs => {
if (configs) {
this.filterValue = configs.data.queryConfiguration[0].search_string;
this.graphQlService.getTweetData().subscribe(data => {
if (data) {
let queryResult: QueryResult = data;
this.graphPayload = this.conversionService.createGraphPayload(queryResult);
this.fitGraph();
this.showData = true;
}
}, (error) => {
console.error('error', error);
}).add(() => {
this.spinner.hide();
});
}
}, (error) => {
console.error('error', error);
}).add(() => {
this.spinner.hide();
});
}
<ngx-graph *ngIf="showData"
class="chart-container"
layout="dagre"
[view]="[1720, 768]"
[showMiniMap]="false"
[zoomToFit$]="zoomToFit$"
[links]="graphPayload.links"
[nodes]="graphPayload.nodes"
>
<ng-template #defsTemplate>
<svg:marker id="arrow" viewBox="0 -5 10 10" refX="8" refY="0" markerWidth="4" markerHeight="4" orient="auto">
<svg:path d="M0,-5L10,0L0,5" class="arrow-head" />
</svg:marker>
</ng-template>
<ng-template #nodeTemplate let-node>
<svg:g class="node" (click)="clickNode(node)">
<svg:rect
[attr.width]="node.dimension.width"
[attr.height]="node.dimension.height"
[attr.fill]="node.data.color"
/>
<svg:text alignment-baseline="central" [attr.x]="10" [attr.y]="node.dimension.height / 2">
{{node.label}}
</svg:text>
</svg:g>
</ng-template>
<ng-template #linkTemplate let-link>
<svg:g class="edge">
<svg:path class="line" stroke-width="2" marker-end="url(#arrow)"></svg:path>
<svg:text class="edge-label" text-anchor="middle">
<textPath
class="text-path"
[attr.href]="'#' + link.id"
[style.dominant-baseline]="link.dominantBaseline"
startOffset="50%"
>
{{link.label}}
</textPath>
</svg:text>
</svg:g>
</ng-template>
</ngx-graph>

Running the Angular Client

Conclusion

  • Twitter API and Developer Application
  • twarc and custom Python code
  • Dgraph Slash GraphQL
  • Angular CLI and ngx-graph

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
John Vester

Information Technology professional with 25+ years expertise in application architecture, design and development. Agile project and team management.