Update: Steem-Exif-Spider-bot New Voting and Comments Queues
Words
252
Reading
2 min
Listen
Play
9y
There are several improvements and features added.
- Comments queue
- Voting queue
- Switch from filesystem streams to in-memory buffers
Comments and Voting queues
In other projects, I have noticed significant efficiency improvements from switching to queues that spread voting/commenting out instead of processing all votes and comments simultaneously. The queue implementations are fairly simple. Just pushing/shifting off an array. It is also managed through a scheduler that shifts items off the array in periodic increments. Ideally, we would want to manage this with an event. That may come in as a future change.
Switching from Filesystem Streams to in-memory Buffers
Prior to this change, JPG data was read via HTTP using streams. The data would be streamed and then written to a file. This file would then be read back in as a buffer. We are essentially skipping a step. Rather than blocking on filesystem I/O for streams, I switched to in-memory buffers. The data is converted from a stream to a buffer and loaded into memory rather than writing to disk first.
Changes
diff --git a/steem-exif-spider-bot/helpers/bot/comment.js b/steem-exif-spider-bot/helpers/bot/comment.js
new file mode 100644
index 0000000..10c2093
--- /dev/null
+++ b/steem-exif-spider-bot/helpers/bot/comment.js
@@ -0,0 +1,61 @@
+const Promise = require('bluebird')
+const steem = require('steem')
+const { user, wif, weight } = require('../../config')
+const schedule = require('node-schedule')
+const Handlebars = require('handlebars')
+const fs = Promise.promisifyAll(require('fs'))
+const path = require('path')
+
+const MINUTE = new schedule.RecurrenceRule();
+MINUTE.second = 1
+
+function loadTemplate(template) {
+ return fs.readFileAsync(template, 'utf8')
+}
+
+
+function execute(comments) {
+
+ if (comments.length() < 1) {
+ return {};
+ }
+
+ const { author, permlink } = comments.shift();
+
+ var context = {
+ }
+
+ return loadTemplate(path.join(__dirname, '..', 'templates', "exif.hb"))
+ .then((template) => {
+ var templateSpec = Handlebars.compile(template)
+ return templateSpec(context)
+ })
+ .then((message) => {
+ var new_permlink = 're-' + author
+ + '-' + permlink
+ + '-' + new Date().toISOString().replace(/[^a-zA-Z0-9]+/g, '').toLowerCase();
+ console.log("Commenting on ", author, permlink, type)
+
+ return steem.broadcast.commentAsync(
+ wif,
+ author, // Leave parent author empty
+ permlink, // Main tag
+ user, // Author
+ new_permlink, // Permlink
+ new_permlink,
+ message, // Body
+ { tags: [], app: "steemit-exif-spider-bot/0.1.0" }
+ ).then((results) => {
+ console.log(results)
+ return results
+ })
+ .catch((err) => {
+ console.log("Error ", err.message)
+ })
+ })
+}
+
+module.exports = {
+ execute
+}
Comment queue implementation. Pulls comments off the queue and posts them to the steem blockchain.
diff --git a/steem-exif-spider-bot/helpers/bot/exif.js b/steem-exif-spider-bot/helpers/bot/exif.js
index bc49a21..738c48a 100644
--- a/steem-exif-spider-bot/helpers/bot/exif.js
+++ b/steem-exif-spider-bot/helpers/bot/exif.js
@@ -17,6 +17,9 @@ module.exports = {
execute
}
+let VOTING = {}
+let COMMENTS = {}
+
function loadTemplate(template) {
return fs.readFileAsync(template, 'utf8')
}
@@ -35,60 +38,32 @@ function processComment(comment) {
}
return [];
})
- .each((image) => {
- if (image.indexOf(".jpg") > -1|| image.indexOf(".JPG") > -1) {
- const dest = tempfile('.jpg');
- try {
- got.stream(image).pipe(fs.createWriteStream(dest))
- .on('close', () => {
- try {
- const input = ExifReader.load(fs.readFileSync(dest));
- const tags = []
- for (let key in input) {
- const value = input[key];
- if (key != "MakerNote"
- && key.indexOf("undefined") < 0
- && key.indexOf("omment") < 0
- && key.indexOf("ersion") < 0) {
- tags.push({ name: key, value: value.value, description: value.description })
- }
- }
-
- reply(comment, tags)
- }
- catch(err) {
- if (err.message == "No Exif data") {
-
- }
- }
- })
- }
- catch (err) {
- console.log("Error ", err)
- }
- finally {
- fs.unlink(dest, (err) => {
- // file deleted
+ .map((image) => {
+ if (image.indexOf(".jpg") > -1 || image.indexOf(".JPG") > -1) {
+ const buffers = [];
+ return got(image, {encoding: null })
+ .then((response) => {
+ console.log("Loading ", image);
+ return ExifReader.load(response.body);
+ })
+ .catch((error) => {
+ console.log("Error ", error);
});
+ }
+ })
+ .filter((tags) => tags ? true : false)
+ .each(input => {
+ const tags = []
+ for (let key in input) {
+ const value = input[key];
+ if (key != "MakerNote"
+ && key.indexOf("undefined") < 0
+ && key.indexOf("omment") < 0
+ && key.indexOf("ersion") < 0) {
+ tags.push({ name: key, value: value.value, description: value.description })
}
}
+ reply(comment, tags)
})
.catch((error) => {
console.log("Error ", error)
@@ -101,51 +76,23 @@ function reply(comment, tags) {
tags: tags
}
-
- return loadTemplate(path.join(__dirname, '..', 'templates', 'exif.hb'))
- .then((template) => {
- var templateSpec = Handlebars.compile(template)
- return templateSpec(context)
- })
- .then((body) => {
- console.log("Body ", body)
- return body;
- })
- .then((body) => {
- var permlink = 're-' + comment.author
- + '-' + comment.permlink
- + '-' + new Date().toISOString().replace(/[^a-zA-Z0-9]+/g, '').toLowerCase();
-
+ return new Promise((resolve, reject) => {
console.log("Replying to ", {author: comment.author, permlink: comment.permlink})
- return steem.broadcast.commentAsync(
- wif,
- comment.author, // Leave parent author empty
- comment.permlink,
- user, // Author
- permlink, // Permlink
- permlink, // Title
- body, // Body
- { "app": "steem-exif-spider-bot/0.1.0" }
- )
- .catch((err) => {
- console.log("Unable to process comment. ", err)
- })
+ COMMENTS.push({ author: comment.author, permlink: comment.permlink })
+
+ return [ comment.author, comment.permlink]
})
- .then((response) => {
- return steem.broadcast.voteAsync(wif, user, comment.author, comment.permlink, weight)
- .then((results) => {
- console.log(results)
- })
- .catch((err) => {
- console.log("Vote failed: ", err)
- })
+ .spread((author, permlink) => {
+ VOTING.push({ author: author, permlink: permlink, weight: weight });
})
.catch((err) => {
console.log("Error loading template ", err)
})
}
-function execute() {
+function execute(voting, comments) {
+ VOTING = voting
+ COMMENTS = comments
steem.api.streamOperations((err, results) => {
return new Promise((resolve, reject) => {
Moving functionality that will be implemented in comment and voting queues. Switching from filesystem streams to in-memory buffers.
diff --git a/steem-exif-spider-bot/helpers/bot/index.js b/steem-exif-spider-bot/helpers/bot/index.js
index 083546c..64b61ac 100644
--- a/steem-exif-spider-bot/helpers/bot/index.js
+++ b/steem-exif-spider-bot/helpers/bot/index.js
@@ -1,6 +1,27 @@
+const voting_queue = [];
+const comment_queue = [];
+
+const voting = {
+ length: () => { return voting_queue.length },
+ push: (obj) => { return voting_queue.push(obj) },
+ pop: () => { return voting_queue.pop() },
+ shift: () => { return voting_queue.shift() },
+ unshift: (obj) => { return voting_queue.unshift(obj) }
+}
+
+const comments = {
+ length: () => { return comment_queue.length },
+ push: (obj) => { return comment_queue.push(obj) },
+ pop: () => { return comment_queue.pop() },
+ shift: () => { return comment_queue.shift() },
+ unshift: (obj) => { return comment_queue.unshift(obj) }
+}
+
function run() {
- return require("./exif").execute();
+ require('./comment').execute(comments)
+ require('./vote').execute(voting)
+ require('./exif').execute(voting, comments)
}
The model for the queue management
diff --git a/steem-exif-spider-bot/helpers/bot/vote.js b/steem-exif-spider-bot/helpers/bot/vote.js
new file mode 100644
index 0000000..58714c8
--- /dev/null
+++ b/steem-exif-spider-bot/helpers/bot/vote.js
@@ -0,0 +1,84 @@
+const Promise = require('bluebird')
+const steem = require('steem')
+const { user, wif, weight } = require('../../config')
+const schedule = require('node-schedule')
+const moment = require('moment');
+
+const MINUTE = new schedule.RecurrenceRule();
+MINUTE.second = 1
+
+const SECONDS_PER_HOUR = 3600
+const PERCENT_PER_DAY = 20
+const HOURS_PER_DAY = 24
+const MAX_VOTING_POWER = 10000
+const DAYS_TO_100_PERCENT = 100 / PERCENT_PER_DAY
+const SECONDS_FOR_100_PERCENT = DAYS_TO_100_PERCENT * HOURS_PER_DAY * SECONDS_PER_HOUR
+const RECOVERY_RATE = MAX_VOTING_POWER / SECONDS_FOR_100_PERCENT
+const DEFAULT_THRESHOLD = 9500
+
+
+function current_voting_power(vp_last, last_vote) {
+ console.log("Comparing %s to %s ", moment().utc().add(7, 'hours').local().toISOString(), moment(last_vote).utc().local().toISOString())
+
+ var seconds_since_vote = moment().utc().add(7, 'hours').local().diff(moment(last_vote).utc().local(), 'seconds')
+ return (RECOVERY_RATE * seconds_since_vote) + vp_last
+}
+
+function time_needed_to_recover(voting_power, threshold) {
+ return (threshold - voting_power) / RECOVERY_RATE
+}
+
+function check_can_vote() {
+ return steem.api.getAccountsAsync([ user]).then((accounts) => {
+ if (accounts && accounts.length > 0) {
+ const account = accounts[0];
+ console.log("Voting threshold for %s: %s", user, DEFAULT_THRESHOLD)
+ console.log("Getting voting power for %d %s", account.voting_power, account.last_vote_time)
+ var voting_power = current_voting_power(account.voting_power, account.last_vote_time)
+ if (voting_power > DEFAULT_THRESHOLD) {
+ return true;
+ }
+ }
+ return false;
+ })
+}
+
+function vote(author, permlink, weight) {
+ return steem.broadcast.voteAsync(
+ wif,
+ user,
+ author,
+ permlink,
+ weight
+ )
+ .then((results) => {
+ console.log("Vote results: ", results)
+ return results;
+ },
+ (err) => {
+ console.log("Vote failed for %s: %s", user, err.message)
+ })
+}
+
+function execute(voting) {
+ schedule.scheduleJob(MINUTE, function() {
+ if (voting.length() < 1) {
+ return {};
+ }
+
+ const { author, permlink, weight } = voting.shift();
+
+ return check_can_vote().then((can_vote) => {
+ if (can_vote) {
+ vote(author, permlink, weight)
+ }
+ else {
+ voting.push({ author, permlink, weight })
+ }
+ })
+ })
+}
+
+module.exports = {
+ execute
+}
This is the module for the voting queue implementation. It periodically pulls votes off a queue and votes them.
diff --git a/steem-exif-spider-bot/package.json b/steem-exif-spider-bot/package.json
index 1abbdbe..8024fd2 100644
--- a/steem-exif-spider-bot/package.json
+++ b/steem-exif-spider-bot/package.json
@@ -31,6 +31,7 @@
"got": "^8.3.0",
"handlebars": "^4.0.11",
"jdataview": "^2.5.0",
+ "node-schedule": "^1.3.0",
"request": "^2.85.0",
"steem": "^0.7.1",
"tempfile": "^2.0.0",
Adding node schedule dependency so the queue can be run using node-schedule
Posted on Utopian.io - Rewarding Open Source Contributors