Initial upload

README.md

@@ -0,0 +1,4 @@
+## music-downloader
+
+Download music from multiple websites without annoying ads and in 1 overview!
+

console.py

@@ -0,0 +1,137 @@
+import sys
+
+import colorama as color
+
+DEBUG_LEVEL = 2
+
+OUTPUT = 0
+DBG_ERROR = 1
+DBG_INFO = 2
+
+LOCKED = False
+
+CACHE = []
+
+color.init(True)
+
+def output(text='', level=OUTPUT, end='\n'):
+    if LOCKED:
+        CACHE.append((text, level, end))
+        return
+
+    if level <= DEBUG_LEVEL:
+
+        if level == 1:
+            text = color.Fore.RED + '[ERROR] ' + text
+        if level == 2:
+            text = color.Fore.YELLOW + '[INFO] ' + text
+
+        print(text, end=end)
+
+
+def ask_input(question, f=str, quit=False):
+    output(color.Style.BRIGHT + color.Fore.CYAN + question, end=': ')
+    _input = input().strip()
+    if quit and _input == 'q':
+        return None
+    else:
+        try:
+            return f(_input)
+        except:
+            output('{0} cannot be casted to type {1}'.format(_input, type(f.__class__)), level=DBG_ERROR)
+
+
+def option_picker(question, options, objects=None, selected=None, quit=False, table=None):
+    output(color.Style.BRIGHT + color.Fore.CYAN + question, end=':\n')
+    width = None
+
+    if type(table) is list:
+        header = []
+        for column in table:
+            _space = '[{:' + str(column[1]) + '}]'
+            header.append(_space.format(column[0]))
+
+        _header = ' '.join(header)
+        width = len(_header)
+        output(color.Style.BRIGHT + _header)
+        output(color.Style.BRIGHT + ('=' * width))
+
+    for i in range(0, len(options)):
+        option = options[i]
+        _row = ''
+
+        if i == selected:
+            _row += color.Style.BRIGHT + '>'
+
+        if objects is not None and table is not None and len(objects) == len(table):
+            for j in range(0, len(objects)):
+                _row += '[{' + str(objects[j]) + ':' + str(table[j][1]) + '}] '
+
+            output(_row.format(__id__=i, x=option).strip())
+
+        else:
+            if i == selected:
+                output('>', end='')
+
+            output('[{0:>2}]'.format(i), end=' ')
+
+            output(option)
+
+    if quit:
+        if width is not None:
+            output(color.Style.BRIGHT + ('=' * width))
+        else:
+            output('==========')
+        output(color.Fore.YELLOW + '[q] cancel')
+
+    return ask_input('Pick option', int, quit=quit)
+
+
+def format_bytes(count):
+    return '{:04.2f}MB'.format(count / 1048576)
+
+
+def lock():
+    global LOCKED
+    LOCKED = True
+
+
+def unlock():
+    global LOCKED, CACHE
+    for print in CACHE:
+        output(print[0], print[1], print[2])
+
+    LOCKED = False
+    CACHE = []
+
+
+class ProgressBar:
+    def __init__(self, total=None, size=100):
+        self.size = size
+        self.total = total
+        self.current = 0
+        lock()
+
+    def destroy(self):
+        print()
+        unlock()
+
+    def report_progress(self, increment=1, progress=None):
+        if progress is not None:
+            self.current = progress
+        else:
+            self.current += increment
+        self.draw()
+
+    def draw(self):
+        if self.total is None:  # Marque
+            pass
+        else:
+            percent = int((self.current / self.total) * 100)
+            _draw = ('\r[{0:' + str(self.size) + '}] {1}%').format('=' * percent, percent)
+
+        sys.stdout.write(_draw)
+        sys.stdout.flush()
+
+
+

main.py

@@ -0,0 +1,152 @@
+import sites as sites
+import console as console
+import os.path
+from settings import Settings
+
+
+def main():
+    console.output('Music downloader by Deben Oldert')
+    console.output()
+
+    directory(os.getenv('MD_SAVEDIR', '~/Music/iTunes/iTunes Media/Automatically Add to iTunes'))
+
+    console.output('Download directory: {0}'.format(Settings.SaveDir))
+    console.output()
+    help()
+    console.output()
+    console.output('Available sites: {0}'.format(', '.join(list(map(lambda x: x.url, sites.available)))))
+    console.output('Selected site is: {0}'.format(sites.available[selected_site].url))
+    console.output()
+
+    while True:
+        if not idle():
+            break
+
+    console.output('Quitting...')
+    exit(0)
+
+
+def choose_site():
+    global selected_site
+    _index = console.option_picker('Available sites',
+                                          list(map(lambda x: x.url, sites.available)),
+                                          selected_site,
+                                          True)
+    if _index is not None:
+        selected_site = _index
+    return True
+
+
+def idle():
+    _command = console.ask_input('Type command')
+
+    for command in commands:
+        if command[0] == _command.lower():
+            return command[1]()
+
+    console.output('Command \'{0}\' not found. Type help to see a list of possible commands'.format(_command))
+    return True
+
+
+def help():
+    console.output('Possible commands are:')
+    for command in commands:
+        console.output('{0}\t{1}'.format(command[0], command[2]))
+    return True
+
+
+def search(cache=None):
+    if cache is None:
+        keywords = console.ask_input('Search for')
+        _sites = []
+        if isinstance(sites.available[selected_site], sites.DefaultSite):
+            _sites = sites.available[1:]
+        else:
+            _sites = [sites.available[selected_site]]
+
+        items = []
+        for site in _sites:
+            console.output('Searching: {0}'.format(site.url))
+            _items = site.perform_search(keywords)
+            console.output('\tFound {0} results'.format(len(_items)))
+            items.extend(_items)
+
+        if len(items) == 0:
+            console.output('No results found')
+            return True
+
+        last(items)
+    else:
+        items = cache
+
+    while True:
+        picked_item = console.option_picker('Pick a song to download',
+                                            items,
+                                            # list(map(lambda x: '[{0:5}] [{1:4}] {2}'.format(x.duration, x.size, x.title), items)),
+                                            quit=True,
+                                            objects=[
+                                                '__id__',
+                                                'x.duration',
+                                                'x.size',
+                                                'x.artist',
+                                                'x.title'
+                                            ],
+                                            table=[
+                                                ('ID', 2),
+                                                ('Time', 5),
+                                                ('Size', 4),
+                                                ('Artist', 50),
+                                                ('Title', 100)
+                                            ])
+        if picked_item is None:
+            break
+        else:
+            item = items[picked_item]
+            console.output('Downloading {0}'.format(item.title))
+
+            item.site.download(item)
+
+            break
+
+    return True
+
+
+def directory(dir=None):
+    if dir is None:
+        dir = console.ask_input('Enter new save directory')
+    Settings.SaveDir = os.path.expanduser(dir)
+    return True
+
+
+def last(items=None):
+    global search_cache
+    if items is not None:
+        search_cache = items
+    else:
+        if search_cache is None:
+            console.output('First search has yet te be made', console.DBG_ERROR)
+        else:
+            search(search_cache)
+    return True
+
+
+def quit():
+    return False
+
+
+selected_site = 0
+search_cache = None
+commands = [
+    ('help', help, 'Display this list'),
+    ('last', last, 'See the last search'),
+    ('save', directory, 'Change the download directory'),
+    ('search', search, 'Search the selected site'),
+    ('sites', choose_site, 'Pick the site to search'),
+    ('quit', quit, 'Quit the program')
+]
+
+# DOWNLOAD_DIR = os.path.expanduser('~/Music/iTunes/iTunes Media/Automatically Add to iTunes')
+# DOWNLOAD_DIR = os.path.expanduser('~/Downloads')
+
+if __name__ == '__main__':
+    main()

settings.py

@@ -0,0 +1,2 @@
+class Settings:
+    SaveDir = None

sites/__init__.py

@@ -0,0 +1,7 @@
+from .czne import Czne
+from .default import DefaultSite
+from .musicteam import MusicTeam
+from .lalamus import Lalamus
+
+
+available = [DefaultSite(), Czne(), MusicTeam(), Lalamus()]

sites/czne.py

@@ -0,0 +1,36 @@
+from sites.default import DefaultSite
+from sites.helper.query import Query
+
+
+class Czne(DefaultSite):
+    def __init__(self):
+        super().__init__()
+        self.url = 'https://nowoscimuzyczne.eu'
+        self.query = Query(self, 'POST')\
+            .add_parameter('do', 'search')\
+            .add_parameter('subaction', 'search')\
+            .add_parameter('story', '{0}')
+        self.structure\
+            .set_container_path('div#dle-content')\
+            .set_item_path('article.short-story')\
+            .set_title_path('div.short-title')\
+            .set_duration_path('div.news-text div.wpisplayer div.total-time') \
+            .set_url_path('div.wpisplayer script:first')
+        self.request\
+            .add_header('Referer', self.url + '/')\
+            .add_header('Origin', self.url)\
+            .add_header('Content-Type', 'application/x-www-form-urlencoded')
+
+    def format_url(self, field):
+        script = field.text()
+
+        _start = 'var link=\''
+        _end = '\';'
+
+        start = script.find(_start)
+        if start > -1:
+            end = script.find(_end)
+            if end > -1 and end > start:
+                return script[start+len(_start):end]
+
+        return None

sites/default.py

@@ -0,0 +1,60 @@
+import console
+from sites.helper.request import Request
+from sites.helper.query import Query
+from sites.helper.structure import Structure
+
+import sites.helper.download as Download
+
+
+class DefaultSite:
+    def __init__(self):
+        self.url = 'All'
+        self.query = Query(self)
+        self.structure = Structure(self)
+        self.request = Request(self)\
+            .add_header('User-Agent',
+                        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 '
+                        '(KHTML, like Gecko) Version/11.1 Safari/605.1.15')
+
+    def perform_search(self, keywords):
+        return self.request.search(keywords)
+
+    def format_title(self, field):
+        if ' - ' in field.text():
+            return field.text().split(' - ')[1]
+        else:
+            return field.text()
+
+    def format_artist(self, field):
+        if ' - ' in field.text():
+            return field.text().split(' - ')[0]
+        else:
+            return ''
+
+    def format_duration(self, field):
+        if field is None:
+            return ''
+        return field.text()
+
+    def format_size(self, field):
+        if field is None:
+            return ''
+        return field.text()
+
+    def format_url(self, field):
+        return field.attr('href')
+
+
+    @staticmethod
+    def download(item):
+        item.format_original_url()
+        # item.original_url = item.site.format_url(item.original_url)
+
+        if 'zippyshare' in item.original_url:
+            Download.zippyaudio(item)
+        elif 'krakenfiles' in item.original_url:
+            Download.krakenfiles(item)
+        elif 'lalamus' in item.original_url:
+            Download.download(item)
+        else:
+            console.output('Don\'t know what to do with link: {}.'.format(item.download_url), level=console.DBG_ERROR)

sites/helper/download.py

@@ -0,0 +1,158 @@
+import console
+from .request import RawRequest
+from pyquery import PyQuery
+from settings import Settings
+
+import math
+import urllib.parse
+import json
+
+
+def download(item, type='GET', parameters=None, headers=None, cookies=None, stream=True):
+    console.output('Requesting dl: {0}'.format(item.download_url), level=console.DBG_INFO)
+    try:
+        if type == 'GET':
+            file = RawRequest.get(item.download_url,
+                                  parameters=parameters,
+                                  headers=headers,
+                                  cookies=cookies,
+                                  stream=stream)
+        else:
+            file = RawRequest.post(item.download_url,
+                                   parameters=parameters,
+                                   headers=headers,
+                                   cookies=cookies,
+                                   stream=stream)
+    except:
+        notExist(item)
+        return
+
+    name = urllib.parse.unquote(item.title)
+
+    if not name.endswith('.mp3'):
+        console.output('Assuming it\'s a mp3 file', console.DBG_INFO)
+        name = name + '.mp3'
+
+    full_name = '{0}/{1}'.format(Settings.SaveDir, name)
+
+    size = file.headers.get('content-length')
+
+    savefileprogress(name, full_name, file, size)
+
+
+def savefileprogress(name, full_name, file, size):
+    if size is not None:
+        size = int(size)
+        console.output('Size: {0}'.format(console.format_bytes(size)))
+    console.output('Saving to: {0}'.format(full_name))
+    with open(full_name, 'wb') as f:
+        progress = 0
+        if size is not None:
+            bar = console.ProgressBar(total=size)
+
+            for chunk in file.iter_content(chunk_size=4096):
+                progress += len(chunk)
+                f.write(chunk)
+                bar.report_progress(progress=progress)
+
+        else:
+            bar = console.ProgressBar()
+            f.write(file.content)
+
+        bar.destroy()
+    console.output('Download of {0} completed!'.format(name))
+
+
+def notExist(item):
+    console.output('{x.title} at {x.download_url} does not exist'.format(x=item))
+
+
+def zippyaudio(item, returnurl=False):
+    console.output('Requesting zippy: {0}'.format(item.original_url), level=console.DBG_INFO)
+
+    item.set_download_url(item.original_url.replace('/wf/', '/v/'))
+
+    body = RawRequest.get(item.download_url)
+
+    pq = PyQuery(body.text)
+
+    script = pq('a#dlbutton').next('script').text()
+
+    if script is None or script == '':
+        return notExist(item)
+
+    _a = 'var a = '
+    _b = 'var b = '
+
+    a = int(script[script.find(_a)+len(_a):script.find(';', script.find(_a))])
+    b = int(script[script.find(_b)+len(_b):script.find(';', script.find(_b))])
+
+    c = math.floor(a/3)
+    d = a % b
+
+    url = script[script.find('/d/'):script.find('";', script.find('/d/'))]
+
+    _replace = '\"+(a + {0}%b)+\"'.format(a)
+
+    url = url.replace(_replace, str(c + d))
+
+    host = urllib.parse.urlparse(item.original_url)
+
+    item.set_download_url('{0}://{1}{2}'.format(host.scheme, host.netloc, url))
+
+    if returnurl:
+        return item.download_url
+    else:
+        download(item)
+
+
+def krakenfiles(item, returnurl=False):
+    console.output('Requesting kraken: {0}'.format(item.original_url), level=console.DBG_INFO)
+
+    link = item.original_url[:item.original_url.find('/waveform.png')]
+    id = link[link.rfind('/')+1:]
+
+    item.set_download_url('https://krakenfiles.com/view/{0}/file.html'.format(id))
+
+    body = RawRequest.get(item.download_url).text
+
+    pq = PyQuery(body)
+
+    form = pq('form#dl-form')
+
+    item.set_download_url('https:{0}'.format(form.attr('action')))
+
+    name = pq('span.dfilename').text()
+
+    parameters = {}
+
+    parameters['token'] = pq('input#dl-token').val()
+
+    headers = {}
+    headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
+    headers['Origin'] = 'https://krakenfiles.com'
+    headers['Referer'] = link
+    headers['X-Requested-With'] = 'XMLHttpRequest'
+    headers['hash'] = id
+
+    cookie = {}
+    cookie['fht_dwn_{0}'.format(id)] = '1'
+
+    json_request = RawRequest.post(item.download_url, parameters=parameters, headers=headers, cookies=cookie)
+
+    _json_data = json.loads(json_request.text)
+
+    if 'url' not in _json_data:
+        notExist(item)
+    else:
+        item.set_download_url(_json_data.get('url'))
+
+        if returnurl:
+            return item.download_url
+        else:
+            download(item,
+                     type='POST',
+                     parameters=parameters,
+                     headers=headers,
+                     cookies=cookie,
+                     stream=True)

sites/helper/item.py

@@ -0,0 +1,24 @@
+import console
+
+
+class Item:
+    def __init__(self, site, title, url, duration='00:00', size=0, artist=None):
+        self.site = site
+        self.original_url = url
+        self.download_url = None
+        self.title = title
+        self.duration = duration
+        self.size = size
+        self.artist = artist
+
+        self.url_formatted = False
+
+    def format_original_url(self):
+        if not self.url_formatted:
+            self.download_url = self.original_url = self.site.format_url(self.original_url)
+            console.output('Setting original URL to: {0}'.format(self.original_url), console.DBG_INFO)
+            self.url_formatted = True
+
+    def set_download_url(self, url):
+        console.output('Setting download url to: {0}'.format(url), console.DBG_INFO)
+        self.download_url = url

sites/helper/query.py

@@ -0,0 +1,16 @@
+class Query:
+    def __init__(self, site, method='GET', path=''):
+        self.site = site
+        self.method = method
+        self.query = ''
+        self.path = path
+
+    def add_parameter(self, key, value='{0}'):
+        self.query += '{0}={1}&'.format(key, value)
+        return self
+
+    def format_query(self, value):
+        if self.query == '':
+            return value
+        else:
+            return self.query.format(value)

sites/helper/request.py

@@ -0,0 +1,83 @@
+import requests
+import urllib.parse
+
+import console
+
+
+class Request:
+    def __init__(self, site):
+        self.site = site
+        self.headers = {}
+        self.cookie = {}
+
+    def add_header(self, key, value):
+        self.headers[key] = value
+        return self
+
+    def add_cookie(self, key, value):
+        self.cookie[key] = value
+        return self
+
+    def search(self, keywords):
+
+        query = self.site.query.format_query(urllib.parse.quote(keywords))
+
+        console.output('Request method: {0}'.format(self.site.query.method), level=console.DBG_INFO)
+
+        if self.site.query.method == 'GET':
+            request = requests.get(
+                '{0}?{1}'.format(self.site.url, query),
+                headers=self.headers,
+                cookies=self.cookie,
+                allow_redirects=True)
+        elif self.site.query.method == 'POST':
+            request = requests.post(
+                self.site.url,
+                data=query,
+                headers=self.headers,
+                cookies=self.cookie,
+                allow_redirects=True)
+        else:  # FLAT
+            request = requests.get(
+                '{0}/{1}/{2}'.format(self.site.url, self.site.query.path, query),
+                headers=self.headers,
+                cookies=self.cookie,
+                allow_redirects=True)
+
+        console.output('Request response: {0} {1}'.format(request.status_code, request.reason), level=console.DBG_INFO)
+
+        return self.site.structure.parse(request.text)
+
+    def parse_url(self, url):
+        o = urllib.parse.urlparse(url)
+
+        if o.query != '':
+            o.query = urllib.parse.parse_qs(o.query)
+
+        return o
+
+
+class RawRequest:
+    @staticmethod
+    def get(link, parameters=None, headers=None, cookies=None, stream=False):
+        request = requests.get(
+            link,
+            data=parameters,
+            headers=headers,
+            cookies=cookies,
+            allow_redirects=True,
+            stream=stream)
+        if request.status_code == 200:
+            return request
+
+    @staticmethod
+    def post(link, parameters=None, headers=None, cookies=None, stream=False):
+        request = requests.post(
+            link,
+            data=parameters,
+            headers=headers,
+            cookies=cookies,
+            allow_redirects=True,
+            stream=stream)
+        if request.status_code == 200:
+            return request

sites/helper/structure.py

@@ -0,0 +1,69 @@
+from pyquery import PyQuery
+
+from sites.helper.item import Item
+
+
+class Structure:
+    def __init__(self, site):
+        self.site = site
+        self.container_path = None
+        self.item_path = None
+        self.item_url_path = None
+        self.item_title_path = None
+        self.item_duration_path = None
+        self.item_size_path = None
+        self.item_artist_path = None
+
+    def set_container_path(self, path):
+        self.container_path = path
+        return self
+
+    def set_item_path(self, path):
+        self.item_path = path
+        return self
+
+    def set_url_path(self, path):
+        self.item_url_path = path
+        return self
+
+    def set_title_path(self, path):
+        self.item_title_path = path
+        return self
+
+    def set_artist_path(self, path):
+        self.item_artist_path = path
+        return self
+
+    def set_duration_path(self, path):
+        self.item_duration_path = path
+        return self
+
+    def set_size_path(self, path):
+        self.item_size_path = path,
+        return self
+
+    def parse(self, html):
+        pq = PyQuery(html)
+
+        pq.make_links_absolute(base_url=self.site.url)
+
+        results = pq(self.container_path).find(self.item_path)
+
+        items = []
+
+        for result in results:
+            result = pq(result)
+            title = result.find(self.item_title_path) if self.item_title_path is not None else None
+            url = result.find(self.item_url_path) if self.item_url_path is not None else None
+            duration = result.find(self.item_duration_path) if self.item_duration_path is not None else None
+            size = result.find(self.item_size_path) if self.item_size_path is not None else None
+
+            if title is not None and url is not None:
+                items.append(Item(self.site,
+                                  self.site.format_title(title),
+                                  url,
+                                  self.site.format_duration(duration),
+                                  self.site.format_size(size),
+                                  self.site.format_artist(title)))
+
+        return items

sites/lalamus.py

@@ -0,0 +1,33 @@
+from sites.default import DefaultSite
+from sites.helper.query import Query
+
+
+class Lalamus(DefaultSite):
+    def __init__(self):
+        super().__init__()
+        self.url = 'https://wwc.lalamus.mobi'
+        self.query = Query(self, 'FLAT', 'music')
+        self.structure\
+            .set_container_path('div.playlist-hash')\
+            .set_item_path('table.song')\
+            .set_title_path('td.title')\
+            .set_duration_path('td.time') \
+            .set_url_path('button.btn_download')
+        self.request\
+            .add_header('Referer', self.url + '/')\
+            .add_header('Origin', self.url)\
+            .add_header('Content-Type', 'application/x-www-form-urlencoded')
+
+    def format_url(self, field):
+        script = field.attr.onclick
+
+        _start = 'openDL(\''
+        _end = '\''
+
+        start = script.find(_start)
+        if start > -1:
+            end = script.find(_end, start+len(_start))
+            if end > -1 and end > start:
+                return '{0}{1}'.format(self.url, script[start+len(_start):end])
+
+        return None

sites/musicid.py

@@ -0,0 +1,28 @@
+from sites.default import DefaultSite
+from sites.helper.query import Query
+import requests
+
+# TODO - Make compatible, add artist path
+class MusicId(DefaultSite):
+    def __init__(self):
+        super().__init__()
+        self.url = 'https://musicid.ru/'
+        self.query = Query(self, 'GET')\
+            .add_parameter('song', '{0}') \
+            .add_parameter('s', 'f')
+        self.structure\
+            .set_container_path('div.results')\
+            .set_item_path('div.chkd')\
+            .set_artist_path('b.artist')\
+            .set_title_path('a.trackLink')\
+            .set_url_path('a.link')
+        self.request\
+            .add_header('Referer', self.url + '/')\
+            .add_header('Origin', self.url)
+
+    def format_url(self, field):
+        link = super().format_url(field)
+
+        request = requests.get(link, cookies=self.request.cookie, allow_redirects=True)
+
+        return request.url

sites/musicteam.py

@@ -0,0 +1,28 @@
+from sites.default import DefaultSite
+from sites.helper.query import Query
+import requests
+
+
+class MusicTeam(DefaultSite):
+    def __init__(self):
+        super().__init__()
+        self.url = 'https://music-team.cc/music'
+        self.query = Query(self, 'GET')\
+            .add_parameter('search', '{0}')
+        self.structure\
+            .set_container_path('div.col-lg-8')\
+            .set_item_path('div.card')\
+            .set_title_path('div.card-body')\
+            .set_url_path('a.leftDownload')
+        self.request\
+            .add_header('Referer', self.url + '/')\
+            .add_header('Origin', self.url)\
+            .add_cookie('visited', 'yes')\
+            .add_cookie('robot', '1')
+
+    def format_url(self, field):
+        link = super().format_url(field)
+
+        request = requests.get(link, cookies=self.request.cookie, allow_redirects=True)
+
+        return request.url