Tony Spiro
March 06, 2017
Medium has become the de-facto platform for publishing online content. With its friction-less UI and viral suggestion engine, it's not hard to understand why it's one of the most popular blogging services. I personally enjoy using Medium to post content to the Cosmic Medium publication. Even though I trust Medium to store all of my content, I wanted a way to save my Medium posts as a backup, or to use in any other future applications. When I couldn't find an available backup application, I decided to build one. In this article, I'm going to show you how to build a Medium backup application using Node.js and Cosmic.
TL;DR
Check out the full source code on GitHub.
Install the app in minutes on Cosmic.
Getting Started
First, we'll need to go over some limitations. Medium makes an RSS feed available to retrieve posts from any personal account, publication or a custom domain but it only allows you to retrieve the last 10 posts. Read this help article from Medium to find out which URL structure to use. It can be https://medium.com/feed/@yourusername or if you have a custom domain https://customdomain.com/feed. If you find that you are only getting partial articles, you may need to go into your Medium account settings and make sure RSS Feed is set to "Full".
Planning the App
We want to be able to do a couple things with our Medium backup app:
1. Manually Import posts from any feed url to our Cosmic Bucket.
2. Add / Remove Cron Jobs which will automatically look for new posts and import them into our Cosmic Bucket.
Building the App
In your text editor of choice, start by adding a package.json file:
{
"dependencies": {
"async": "^2.1.4",
"body-parser": "^1.15.2",
"cosmicjs": "^2.35.0",
"express": "^4.14.0",
"hogan-express": "^0.5.2",
"nodemon": "^1.11.0",
"request": "^2.79.0",
"slug": "^0.9.1",
"xml2js": "^0.4.17"
},
"scripts": {
"start": "node app.js",
"development": "nodemon app.js -e js, views/html"
}
}Then run the following command:
npm install
Next create a new file titled app.js and add the following:
// app.js
var express = require('express')
var async = require('async')
var bodyParser = require('body-parser')
var app = express()
app.use(bodyParser.json())
var hogan = require('hogan-express')
app.engine('html', hogan)
app.set('port', (process.env.PORT || 3000))
app.use('/', express.static(__dirname + '/public/'))
// Config
var bucket_slug = process.env.COSMIC_BUCKET || 'medium-backup'
var config = {
cron_interval: process.env.CRON_INTERVAL || 3600000,
bucket: {
slug: bucket_slug,
read_key: process.env.COSMIC_READ_KEY || '',
write_key: process.env.COSMIC_WRITE_KEY || ''
},
url: process.env.URL || 'http://localhost:' + app.get('port')
}
// Routes
require('./routes/index.js')(app, config)
require('./routes/import-posts.js')(app, config, async)
require('./routes/add-crons.js')(app, config, async)
require('./routes/delete-cron.js')(app, config)
app.listen(app.get('port'))Notice we are using Express for our web framework, we have set our configuration to point to our Cosmic Bucket and added a few routes to handle our home page (index.js), our post import route and our routes to handle the cron adding and deleting.
Our index.js file is pretty simple. Just add the following:
// index.js
module.exports = function(app, config) {
app.get('/', function(req, res) {
var Cosmic = require('cosmicjs')
Cosmic.getObjectType(config, { type_slug: 'crons' }, function(err, response) {
res.locals.crons = response.objects.all
res.locals.bucket_slug = config.bucket.slug
res.render('index.html')
})
})
}Basically we are calling the Cosmic API to see if we have any crons saved, then rendering our index.html file located in the views folder.
Import Posts Manually
Next, let's build the import posts functionality. Create a file titled import-posts.js and add the following:
// import-posts.js
module.exports = function(app, config, async) {
app.post('/import-posts', function(req, res) {
var Cosmic = require('cosmicjs')
var request = require('request')
var slug = require('slug')
var parseString = require('xml2js').parseString
var feed_url = req.body.feed_url
var bucket_slug = req.body.bucket_slug
var cosmic_config = {
bucket: {
slug: bucket_slug,
read_key: process.env.COSMIC_READ_KEY || '',
write_key: process.env.COSMIC_WRITE_KEY || ''
}
}
request(feed_url, function (error, response, body) {
if (!error && response.statusCode == 200) {
parseString(body, function (err, result) {
var posts = result.rss.channel[0].item
var posts_imported = []
async.eachSeries(posts, (post, callback) => {
var title = 'Post'
if (post.title)
title = post.title[0]
var content, published_at, modified_at, categories, created_by, medium_link;
if (post.description)
content = post.description[0]
if (post['content:encoded'])
content = post['content:encoded'][0]
if (post['pubDate'])
published_at = post['pubDate'][0]
if (post['atom:updated'])
modified_at = post['atom:updated'][0]
if (post['category'])
categories = post['category']
if (post['dc:creator'])
created_by = post['dc:creator'][0]
if (post['link'])
medium_link = post['link'][0]
// Test if object available
Cosmic.getObject(cosmic_config, { slug: slug(title) }, function(err, response) {
if (response && response.object) {
// already added
return callback()
} else {
var params = {
title: title,
slug: slug(title),
content: content,
type_slug: 'posts',
write_key: config.bucket.write_key,
metafields: [
{
key: 'published_at',
title: 'Published At',
value: published_at
},
{
key: 'modified_at',
title: 'Modified At',
value: modified_at
},
{
key: 'created_by',
title: 'Created By',
value: created_by
},
{
key: 'medium_link',
title: 'Medium Link',
value: medium_link
}
]
}
if (categories) {
var tags = ''
categories.forEach(category => {
tags += category + ', '
})
params.metafields.push({
key: 'tags',
title: 'Tags',
value: tags
})
}
Cosmic.addObject(cosmic_config, params, function(err, response) {
if (response)
posts_imported.push(post)
callback()
})
}
})
}, () => {
if (!posts_imported.length) {
res.status(500).json({ error: 'There was an error with this request.' })
}
res.json({
bucket_slug: config.bucket.slug,
posts: posts_imported
})
})
})
} else {
res.status(500).json({ error: 'feed_url' })
}
})
})
}This file does most of the work in our Medium backup app. What's happening here is:
1. First, we post feed_url and bucket_slug.
2. The RSS feed is accessed with the request module and the data is parsed and converted to JSON for easy management.
3. Then we loop through all of the posts and check if the post already exists in our Cosmic Bucket. If it exists, do nothing. It it doesn't exist:
4. Create the object in our Cosmic Bucket
This saves the title, content (HTML also!), published date / time, author, Medium link and even tags to your Cosmic Bucket.
Import Posts Automatically
This is great for our manual import, next let's create the ability to automatically import my latest articles automatically on a timer. Create a couple files titled add-crons.js and delete-cron.jsto add / remove our cron jobs:
// add-cron.js
module.exports = function(app, config, async) {
app.post('/add-crons', function(req, res) {
var Cosmic = require('cosmicjs')
var slug = require('slug')
var crons = req.body
async.eachSeries(crons, (cron, callback) => {
var params = {
title: cron.title,
slug: slug(cron.title),
type_slug: 'crons',
write_key: config.bucket.write_key,
metafields: [
{
key: 'feed_url',
title: 'Feed URL',
value: cron.feed_url
},
{
key: 'bucket_slug',
title: 'Bucket Slug',
value: cron.bucket_slug
}
]
}
Cosmic.addObject(config, params, function(err, response) {
callback()
})
}, () => {
res.json({
status: "success"
})
})
})
}// delete-cron.js
module.exports = function(app, config, async) {
app.post('/delete-cron', function(req, res) {
var Cosmic = require('cosmicjs')
var slug = req.body.slug
var params = {
write_key: config.bucket.write_key,
slug: slug
}
Cosmic.deleteObject(config, params, function(err, response) {
res.json({
status: "success"
})
})
})
}Next create a file titled crons.js and add the following:
// crons.js
module.exports = function(app, config, async) {
var Cosmic = require('cosmicjs')
var request = require('request')
var locals = {}
async.series([
callback => {
Cosmic.getObjectType(config, { type_slug: 'crons' }, function(err, response) {
locals.crons = response.objects.all
callback()
})
},
callback => {
if (locals.crons) {
async.eachSeries(locals.crons, (cron, callbackEach) => {
var feed_url = cron.metadata.feed_url
var bucket_slug = cron.metadata.bucket_slug
var params = {
feed_url: feed_url,
bucket_slug: bucket_slug
}
var options = {
url: config.url + '/import-posts',
json: params
}
request.post(options, (err, httpResponse, body) => {
if (err) {
return console.error('upload failed:', err)
}
console.log('Successful! Server responded with:', body)
callbackEach()
});
})
}
}
])
}What's happening is we check our Bucket for any Objects in the "Crons" Object Type. If any are found, we loop through all of these and POST to the import-posts endpoint in our app and import the latest posts.
Next we will want to set this to run on a timer, change our app.js to look like the following (added the cron at the bottom):
// app.js
var express = require('express')
var async = require('async')
var bodyParser = require('body-parser')
var app = express()
app.use(bodyParser.json())
var hogan = require('hogan-express')
app.engine('html', hogan)
app.set('port', (process.env.PORT || 3000))
app.use('/', express.static(__dirname + '/public/'))
// Config
var bucket_slug = process.env.COSMIC_BUCKET || 'medium-backup'
var config = {
cron_interval: process.env.CRON_INTERVAL || 3600000,
bucket: {
slug: bucket_slug,
read_key: process.env.COSMIC_READ_KEY || '',
write_key: process.env.COSMIC_WRITE_KEY || ''
},
url: process.env.URL || 'http://localhost:' + app.get('port')
}
// Routes
require('./routes/index.js')(app, config)
require('./routes/import-posts.js')(app, config, async)
require('./routes/add-crons.js')(app, config, async)
require('./routes/delete-cron.js')(app, config)
// Crons
var getCrons = require('./routes/crons.js')
setInterval(() => getCrons(app, config, async), config.cron_interval) // every 60 minutes
app.listen(app.get('port'))And that's pretty much it for our processing files. For the display and simple jQuery frontend check out the index.html file.
Conclusion
I hope you enjoyed this article on how to build a Medium backup app. To begin backing up your Medium posts automatically, install this app on Cosmic in minutes. Let me know if you have any questions about this app or Cosmic reach out
to us on Twitter or join us in the Cosmic Slack channel.
