Skip to content

Commit

Permalink
seperated files
Browse files Browse the repository at this point in the history
  • Loading branch information
@refik
refik committed Jun 24, 2014
1 parent 9acaa25 commit 273d5f9
Showing 1 changed file with 0 additions and 328 deletions.
328 changes: 0 additions & 328 deletions main.go
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,12 @@
package main

import (
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
"os/user"
"path"
"strconv"
"sync"
"time"
)
Expand All @@ -27,13 +20,6 @@ var (
CheckInterval = flag.Int("check-minutes", 5, "check interval of remote files in put.io")
)

const (
ApiUrl = "https://api.put.io/v2/"
DownloadExtension = ".ptdownload"
MaxConnection = 5
ChunkSize int64 = 32 * 1024
)

// Globals

var (
Expand All @@ -43,111 +29,6 @@ var (
TotalFilesSize int64
)

// Putio api response types

type FilesResponse struct {
Files []File `json:"files"`
}

type FileResponse struct {
File File `json:"file"`
}

type File struct {
Id int `json:"id"`
Name string `json:"name"`
ContentType string `json:"content_type"`
Size int64 `json:"size"`
}

func (file *File) DownloadUrl() string {
method := "files/" + strconv.Itoa(file.Id) + "/download"
return MakeUrl(method, map[string]string{})
}

// Api request functions

func ParamsWithAuth(params map[string]string) string {
newParams := url.Values{}

for k, v := range params {
newParams.Add(k, v)
}

newParams.Add("oauth_token", *AccessToken)
return newParams.Encode()
}

func MakeUrl(method string, params map[string]string) string {
newParams := ParamsWithAuth(params)
return ApiUrl + method + "?" + newParams
}

func GetRemoteFolderId() (folderId int, err error) {
// Making sure this folder exits. Creating if necessary
// and updating global variable
files, err := FilesListRequest(0)
if err != nil {
return
}

// Looping through files to get the putitin folder
for _, file := range files {
if file.Name == *RemoteFolderName {
log.Println("Found remote folder")
return file.Id, nil
}
}

postUrl := MakeUrl("files/create-folder", map[string]string{})
postValues := url.Values{}
postValues.Add("name", *RemoteFolderName)
postValues.Add("parent_id", "0")
resp, err := http.PostForm(postUrl, postValues)
if err != nil {
return
}
defer resp.Body.Close()

body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}

fileResponse := FileResponse{}
err = json.Unmarshal(body, &fileResponse)
if err != nil {
return
}

return fileResponse.File.Id, nil
}

func FilesListRequest(parentId int) (files []File, err error) {
// Preparing url
params := map[string]string{"parent_id": strconv.Itoa(parentId)}
folderUrl := MakeUrl("files/list", params)

resp, err := http.Get(folderUrl)
if err != nil {
return
}

body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
defer resp.Body.Close()

filesResponse := FilesResponse{}
err = json.Unmarshal(body, &filesResponse)
if err != nil {
return
}

return filesResponse.Files, nil
}

// Download functions

func WalkAndDownload(parentId int, folderPath string, runWg *sync.WaitGroup, reportCh chan Report) {
Expand Down Expand Up @@ -185,215 +66,6 @@ func WalkAndDownload(parentId int, folderPath string, runWg *sync.WaitGroup, rep
}
}

// Downloads the given range. In case of an error, sleeps for 10s and tries again.
func DownloadRange(file *File, fp *os.File, offset int64, size int64, rangeWg *sync.WaitGroup, chunkIndex bitField, reportCh chan Report) {
defer rangeWg.Done()
reportCh <- Report{ToDownload: size}
newOffset := offset
lastByte := offset + size // The byte we won't be getting
lastIndex := lastByte/ChunkSize - 1 // The last index we'll fill

// Creating a custom request because it will have Range header in it
req, _ := http.NewRequest("GET", file.DownloadUrl(), nil)

rangeHeader := fmt.Sprintf("bytes=%d-%d", offset, lastByte-1)
req.Header.Add("Range", rangeHeader)

// http.DefaultClient does not copy headers while following redirects
client := &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
req.Header.Add("Range", rangeHeader)
return nil
},
}

resp, err := client.Do(req)
if err != nil {
log.Println(err)
return
}
defer resp.Body.Close()

buffer := make([]byte, ChunkSize)
for {
nr, er := io.ReadFull(resp.Body, buffer)
if nr > 0 {
nw, ew := fp.WriteAt(buffer[0:nr], newOffset)
nWritten := int64(nw)
newOffset += nWritten
currentIndex := newOffset/ChunkSize - 1
if currentIndex == lastIndex && newOffset != lastByte {
// dont mark the last bit done without finishing the whole range
} else {
chunkIndex.Set(currentIndex)
fp.WriteAt(chunkIndex, file.Size)
}
reportCh <- Report{Downloaded: nWritten}
if ew != nil {
log.Println(ew)
return
}
}
if er == io.EOF || er == io.ErrUnexpectedEOF {
return
}
if er != nil {
log.Println(er)
return
}
}
}

func DownloadFile(file File, path string, runWg *sync.WaitGroup, reportCh chan Report) error {
defer runWg.Done()
downloadPath := path + DownloadExtension
chunkIndex := bitField(make([]byte, file.Size/ChunkSize/8+1))
resume := false
var fp *os.File

// Creating a waitgroup to wait for all chunks to be
// downloaded before returning
var rangeWg sync.WaitGroup

// Checking whether previous download exists
if _, err := os.Stat(downloadPath); err != nil {
log.Println("Downloading:", file.Name)
fp, err = os.Create(downloadPath)
if err != nil {
log.Println(err)
return err
}
defer fp.Close()

// Allocating space for the file
err = FillWithZeros(fp, file.Size+int64(len(chunkIndex)))
if err != nil {
log.Println(err)
return err
}
} else {
log.Println("Resuming:", file.Name)
resume = true
fp, err = os.OpenFile(downloadPath, os.O_RDWR, 0755)
if err != nil {
log.Println(err)
return err
}
defer fp.Close()
_, err = fp.ReadAt(chunkIndex, file.Size)
if err != nil {
log.Println(err)
return err
}
}

rangeSize := file.Size / MaxConnection
excessBytes := file.Size % MaxConnection

offset := int64(0)
for i := 0; i < MaxConnection; i++ {
rangeCustomOffset := offset
offset += rangeSize
rangeCustomSize := rangeSize
if i == MaxConnection-1 {
// Add excess bytes to last connection
rangeCustomSize = rangeSize + excessBytes
}
if resume {
// Adjusting range for previously downloaded file
startIndex := rangeCustomOffset / ChunkSize
limitIndex := (rangeCustomOffset + rangeSize) / ChunkSize

zIndex, err := chunkIndex.GetFirstZeroIndex(startIndex, limitIndex)
if err == nil {
// This range in not finished yet
zByteIndex := zIndex * ChunkSize
if zByteIndex > rangeCustomOffset {
rangeCustomSize -= zByteIndex - rangeCustomOffset
rangeCustomOffset = zByteIndex
}

} else {
continue
}
}
rangeWg.Add(1)
go DownloadRange(&file, fp, rangeCustomOffset, rangeCustomSize, &rangeWg, chunkIndex, reportCh)
}

// Waiting for all chunks to be downloaded
rangeWg.Wait()

// Verifying the download, some ranges may not be finished
_, err := chunkIndex.GetFirstZeroIndex(0, file.Size/ChunkSize)
if err == nil {
// All chunks are not downloaded
log.Println("All chunks are not downloaded, closing file for dowload:", file.Name)
fp.Close()
return nil
}

// Renaming the file to correct path
fp.Truncate(file.Size)
fp.Close()
err = os.Rename(downloadPath, path)
if err != nil {
log.Println(err)
return err
}

log.Println("Download completed:", file.Name)
return nil
}

// Utility functions

func FillWithZeros(fp *os.File, remainingWrite int64) error {
var nWrite int64 // Next chunk size to write
zeros := make([]byte, ChunkSize)
for remainingWrite > 0 {
// Checking whether there is less to write than chunkSize
if remainingWrite < ChunkSize {
nWrite = remainingWrite
} else {
nWrite = ChunkSize
}

_, err := fp.Write(zeros[0:nWrite])
if err != nil {
return err
}

remainingWrite -= nWrite
}
return nil
}

type bitField []byte

// Set bit i. 0 is the most significant bit.
func (b bitField) Set(i int64) {
div, mod := divMod(i, 8)
b[div] |= 1 << (7 - uint32(mod))
}

// Test bit i. 0 is the most significant bit.
func (b bitField) Test(i int64) bool {
div, mod := divMod(i, 8)
return (b[div] & (1 << (7 - uint32(mod)))) > 0
}

func (b bitField) GetFirstZeroIndex(offset, limit int64) (int64, error) {
for pos := offset; pos < limit; pos++ {
if bit := b.Test(pos); bit == false {
return pos, nil
}
}
return 0, errors.New("No zero in given range")
}

func divMod(a, b int64) (int64, int64) { return a / b, a % b }

func StartWalkAndDownloadClearReports(RemoteFolderId int, reportCh chan Report) {
TotalFilesSize = 0
TotalDownloaded = 0
Expand Down

0 comments on commit 273d5f9

Please sign in to comment.