P2P File Sharing with Fire★

I have been expanding the functionality of Fire★ based on use cases. The first was creating the building blocks for a chat App, the next was the basics for a distributed drawing App. I thought the next obvious use case was a file transfer app.

Demo

Primitives

At a minimum I needed a way to open files, split the files into chunks, send them over the wire, and reassemble the file and save it. For safety the file open API is very simple and only allows the user to select a file…

[code language=”ruby”]
file = app:open_bin_file()
[/code]

And saving a file…

[code language=”ruby”]
app:save_bin_file("my_file", bin_data)
[/code]

Since Lua doesn’t have a native type to deal with bytes, I had to create one that allows to extract a sub array of bytes, get the size, and also append bytes.

Algorithm

For sending the file across the network, I needed several types of messages. The basic algorithm I decided on was that the sender would send a file initiation message with metadata about the file, including how many chunks. Then the receiver would ask the sender for a specific chunk. When the chunk arrived, it would be appended to the working set on the receiver end, where the receiver would then ask for the next chunk.

This pull model provides an obvious advantage, which is that it allows a limited rate of transfer allowing multiplexing with other messages that would be sent. So you can keep chatting, or drawing, or whatever while the file transferred.

Another obvious advantage to the pull model is that you can send multiple files at a time and they will be properly multiplexed. And if you have multiple people you are sending the file to, each one can receive it at their own speed.

The file data structure looks like this

[code language=”ruby”]
local file = {
id = send_id,
name=file:name(),
data=file:data(),
size=file:size(),
chunk=0,
chunks = math.ceil(file:size() / CHUNK),
ch = {},
mode=1}
[/code]

Sending the initiation message looks like this

[code language=”ruby”]
function send_start_file(f)
local m = app:message()
m:set("t","sf")
m:set("name", f.name )
m:set("id", f.id)
m:set("size", f.size)
m:set("chunks", f.chunks)
app:send(m)
end
[/code]

When the receiver gets the message, the following function handles it.

[code language=”ruby”]
app:when_message_received("got")
function got(m)
local t = m:get("t")

if t == "sf" then
got_start_file(m)
elseif t == "gc" then
got_get_chunk(m)
elseif t == "sc" then
got_chunk(m)
end
end
[/code]

This function is the most important one. “got_start_file” adds the metadata to its set and asks for the first chunk. Which the sender will then get a “gc” message which then “got_get_chunk” is called where the sender will then construct a chunk message and send it. The receiver will then get the “sc” message and call “got_chunk” function which will append the chunk to the working set and ask for the next one.

This file transfer application is not built into Fire★, but is built on basic building blocks. You can imagine many ways to improve this application such as adding a pause or a cancel button. This transfer App is part of the standard distribution of packaged apps.

Contribute

I am excited with the progress Fire★ has made. I am adding a breadth of features first to make this platform maximally useful. You can now chat, draw, and transfer files with multiple peers via p2p where all messages are encrypted and private.

Like what you see? Fire★ is GPLv3, please contribute using Github

or email firestr.dev@gmail.com

Full Source

[code language=”ruby”]
s = app:button("send")
app:place(s, 0,0)
app:height(100)
row=1
send_id = 1
CHUNK=1024 * 40

sfiles = {}
gfiles = {}

s:when_clicked("send()")

function send()
local file = app:open_bin_file()

if not file:good() then
return
end

local nf = {
id = send_id,
name=file:name(),
data=file:data(),
size=file:size(),
chunk=0,
chunks = math.ceil(file:size() / CHUNK),
ch = {},
mode=1}
send_id = send_id + 1
add_sfile(nf)
send_start_file(nf)
end

function format_percent(p)
p = p * 1000
p = math.ceil(p)
p = p / 10
return p .. "%"
end

function g_percent(f)
return format_percent(f.chunk / f.chunks)
end

function s_percent(f)
local sc = f.chunks
for k, v in pairs(f.ch) do
if v < sc then
sc = v
end
end
return format_percent( sc / f.chunks)
end

function s_status(f)
local s = ""
local mode = f.mode
local p = s_percent(f)
if mode == 1 then
s = "sending…"
elseif mode == 2 then
s = "… " .. p
elseif mode == 3 then
s = "done"
end
return f.name .. " ".. s
end

function g_status(f)
local s = ""
local mode = f.mode
local p = g_percent(f)
if mode == 1 then
s = "getting…"
elseif mode == 2 then
s = "… " .. p
elseif mode == 3 then
s = "done"
end
return f.name .. " ".. s
end

function update_s_status(fd)
local lb = fd.label
local bt = fd.bt
lb:set_text(s_status(fd.file))
end

function update_g_status(fd)
local lb = fd.label
local bt = fd.bt
lb:set_text(g_status(fd.file))
if fd.file.mode == 3 then
bt:enable()
bt:set_text("save")
bt:when_clicked("save_file_by_id(\"".. fd.id .."\")")
end
end

function add_sfile(f)
local i = f.id
row = row + 1

local cv = app:grid()
app:place(cv, row, 0)
local fl= app:label(s_status(f))
cv:place(fl, 0, 0)
local bt = nil

app:grow()
local fd = {id=i, file=f, label=fl, cv=cv}
sfiles[i] = fd
end

function add_gfile(f)
local i = f.id
row = row + 1

local cv = app:grid()
app:place(cv, row, 0)
local fl= app:label(g_status(f))
cv:place(fl, 0, 0)

local bt= app:button("save")
bt:disable()
cv:place(bt, 0, 1)

app:grow()
local fd = {id=i, file=f, label=fl, bt=bt, cv=cv}
gfiles[i] = fd
end

function send_start_file(f)
local m = app:message()
m:set("t","sf")
m:set("name", f.name )
m:set("id", f.id)
m:set("size", f.size)
m:set("chunks", f.chunks)
app:send(m)
end

function got_start_file(m)

local n = m:get("name")
local orig_id = m:get("id") + 0
local from = m:from()
local id = from:id() .. "_" .. orig_id
local chunks = m:get("chunks") + 0
local size = m:get("size")
local nf = {
from = from,
orig_id = orig_id,
id=id,
name=n,
data=d,
chunks=chunks,
chunk=-1,
size = size,
mode=4}

add_gfile(nf)
send_get_chunk(nf)
end

function send_get_chunk(f)
local m = app:message()
local chunk = f.chunk + 1
m:set("t","gc")
m:set("id", f.orig_id)
m:set("chunk", chunk)

app:send_to(f.from, m)
end

function got_get_chunk(m)
local id = m:get("id") + 0
local chunk = m:get("chunk") + 0
local f = sfiles[id].file

send_chunk(m:from(), f, chunk)
end

function sent_all(f)
local all = true
for k,v in pairs(f.ch) do
if v < (f.chunks – 1) then
all = false
end
end
return all
end

function send_chunk(to, f, c)
local b = c * CHUNK
if b >= f.size then return end
local s = math.min(CHUNK, (f.size – b))
local ch = f.data:sub(b, s)

f.ch[to:id()] = c
if sent_all(f) then
f.mode = 3
else
f.mode = 2
end

local m = app:message()
m:set("t", "sc")
m:set("id", f.id)
m:set("chunk", c)
m:set_bin("data", ch)
app:send_to(to, m)
update_s_status(sfiles[f.id])
end

function got_chunk(m)
local orig_id = m:get("id") + 0
local id = m:from():id() .. "_" .. orig_id
local chunk = m:get("chunk") + 0
local chunk_data = m:get_bin("data")

local fd = gfiles[id]
local file = fd.file
if file.data == nil then
file.data = chunk_data
else
file.data:append(chunk_data)
end
file.chunk = chunk

local last_chunk = file.chunks – 1

if file.chunk == last_chunk then
file.mode = 3
update_g_status(fd)
return
end

file.mode = 2
update_g_status(fd)

send_get_chunk(file)

end

app:when_message_received("got")
function got(m)

local t = m:get("t")

if t == "sf" then
got_start_file(m)
elseif t == "gc" then
got_get_chunk(m)
elseif t == "sc" then
got_chunk(m)
end
end

function save_file_by_id(gid)
local f = gfiles[gid].file
save_file(f)
end

function save_file(f)
f.saved = 1
app:save_bin_file(f.name, f.data)
end
[/code]

Subscribe to Maxim Khailo's Writing

Don’t miss out on the latest issues. Sign up now to get access to the library of members-only issues.
jamie@example.com
Subscribe