构造自己的python模块
15 minute read
0xff better
http://blog.csdn.net/gqtcgq/article/details/49255995
http://blog.useasp.net/archive/2014/09/09/packaging-python-libraries-and-upload-to-pypi-python-package-index.aspx
上传包含文件夹时
https://stackoverflow.com/questions/7522250/how-to-include-package-data-with-setuptools-distribute
制作的模块有其他模块的依赖时
http://blog.topspeedsnail.com/archives/7380
0x00 about
目的:便于关键常用代码的可重用性
用法:
pip3 install exp10it
from exp10it import *
说明:最新模块见如下链接
0x01 api
def figlet2file(logo_str,file_abs_path,print_or_not):
#输出随机的logo文字到文件或屏幕,第二个参数为任意非文件的参数时(eg.0,1,2),只输出到屏幕
#apt-get install figlet
#man figlet
#figure out which is the figlet's font directory
#my figlet font directory is:
#figlet -I 2,output:/usr/share/figlet
def oneline2nline(oneline,nline,file_abs_path):
#将文件中的一行字符串用多行字符串替换,调用时要将"多行字符串的参数(第二个参数)"中的换行符设置为\n
def lin2win(file_abs_path):
#将linux下的文件中的\n换行符换成win下的\n\n换行符
def get_all_file_name(folder,ext_list):
#得到文件夹下的所有文件名的绝对路经,第二个参数为要求的后缀列表
def save2github(file_abs_path,repo_name,comment):
#将文件上传到github
#arg1:文件绝对路经
#arg2:远程仓库名
#提交的commit注释
def get_title_from_uri(uri):
#得到uri对应的title```
def get_uris_from_file(file):
#从文件中获取所有uri### 0x02 module
def get_title_from_file(file):
#等到文件中的所有uri对应的title
def newscript():
#快速写脚本,加logo,写完后可选上传到github
def figlet2file(logo_str,file_abs_path,print_or_not):
#输出随机的logo文字到文件或屏幕,第二个参数为任意非文件的参数时(eg.0,1,2),只输出到屏幕
#apt-get install figlet
#man figlet
#figure out which is the figlet's font directory
#my figlet font directory is:
#figlet -I 2,output:/usr/share/figlet
def oneline2nline(oneline,nline,file_abs_path):
#将文件中的一行字符串用多行字符串替换,调用时要将"多行字符串的参数(第二个参数)"中的换行符设置为\n
def lin2win(file_abs_path):
#将linux下的文件中的\n换行符换成win下的\n\n换行符
def get_all_file_name(folder,ext_list):
#exp_list为空时,得到目录下的所有文件名,不返回空文件夹名
#返回结果为文件名列表,不是完全绝对路径名
#eg.folder="/root"时,当/root目录下有一个文件夹a,一个文件2.txt,a中有一个文件1.txt
#得到的函数返回值为['a/1.txt','2.txt']
def save2github(file_abs_path,repo_name,comment):
#将文件上传到github
#arg1:文件绝对路经
#arg2:远程仓库名
#提交的commit注释
def get_title_from_uri(uri):
#得到uri对应的title
def get_uris_from_file(file):
#从文件中获取所有uri
def get_title_from_file(file):
#等到文件中的所有uri对应的title
def check_file_has_logo(file_abs_path):
a = '### blog: http://3xp10it.github.io'
def newscript():
#快速写脚本,加logo,写完后可选上传到github
def blog():
#便捷写博客(jekyll+github)函数
0x02 source
#############################################################
### ___ / _ | _ \ _) |
### _ \\ \ / __ \ | | | | __|
### ) |` < | | | | | | |
### ____/ _/\_\ .__/ _|\___/ _|\__|
### _|
###
### name: exp10it.py
### function: my module
### date: 2016-08-05
### author: quanyechavshuo
### blog: http://3xp10it.github.io
#############################################################
# -*- coding: utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
#upon 4 lines for chinese support
import os
import re
import random
def figlet2file(logo_str,file_abs_path,print_or_not):
#输出随机的logo文字到文件或屏幕,第二个参数为任意非文件的参数时(eg.0,1,2),只输出到屏幕
#apt-get install figlet
#man figlet
#figure out which is the figlet's font directory
#my figlet font directory is:
#figlet -I 2,output:/usr/share/figlet
try:
f=os.popen("figlet -I 2")
all=f.readlines()
f.close()
figlet_font_dir=all[0][:-1]
except:
os.system("apt-get install figlet")
f=os.popen("figlet -I 2")
all=f.readlines()
f.close()
figlet_font_dir=all[0][:-1]
all_font_name_list=get_all_file_name(figlet_font_dir,['tlf','flf'])
random_font=random.choice(all_font_name_list)
unsucceed=os.system("figlet -t -f %s %s > /tmp/3" % (random_font,logo_str))
if(unsucceed==1):
print "something wrong with figlet,check the command in python source file"
try:
os.system("cat /tmp/3 >> %s" % file_abs_path)
except:
pass
if(print_or_not==True):
os.system("cat /tmp/3")
os.system("rm /tmp/3")
def oneline2nline(oneline,nline,file_abs_path):
#将文件中的一行字符串用多行字符串替换,调用时要将"多行字符串的参数(第二个参数)"中的换行符设置为\n
tmpstr=nline.replace('\n','\\\n')
os.system("sed '/%s/c\\\n%s' %s > /tmp/1" % (oneline,tmpstr,file_abs_path))
os.system("cat /tmp/1 > %s && rm /tmp/1" % file_abs_path)
pass
def lin2win(file_abs_path):
#将linux下的文件中的\n换行符换成win下的\n\n换行符
import sys
input_file=file_abs_path
f=open(input_file,"r+")
urls=f.readlines()
f.close()
os.system("rm %s" % file_abs_path)
f1=open(file_abs_path,"a+")
for url in urls:
print url[0:-1]
#print url is different with print url[0:-1]
#print url[0:-1] can get the pure string
#while print url will get the "unseen \n"
#this script can turn a file with strings
#end with \n into a file with strings end
#with \r\n to make it comfortable move the
#txt file from *nix to win,coz the file with
#strings end with \n in *nix is ok for human
#to see "different lines",but this kind of file
#will turn "unsee different lines" in win
f1.write(url[0:-1]+"\r\n")
f1.close()
#attention:
#由于此处tmp_get_file_name_value和tmp_all_file_name_list在函数外面,so
#在其他代码中调用get_all_file_name()时要用from name import *,不用import name,否则不能调用到get_all_file_name的功能
tmp_get_file_name_value=0
tmp_all_file_name_list=[]
def get_all_file_name(folder,ext_list):
#exp_list为空时,得到目录下的所有文件名,不返回空文件夹名
#返回结果为文件名列表,不是完全绝对路径名
#eg.folder="/root"时,当/root目录下有一个文件夹a,一个文件2.txt,a中有一个文件1.txt
#得到的函数返回值为['a/1.txt','2.txt']
global tmp_get_file_name_value
global root_dir
global tmp_all_file_name_list
tmp_get_file_name_value+=1
if tmp_get_file_name_value==1:
if folder[-1]=='/':
root_dir=folder[:-1]
else:
root_dir=folder
allfile=os.listdir(folder)
for each in allfile:
each_abspath=os.path.join(folder,each)
if os.path.isdir(each_abspath):
get_all_file_name(each_abspath,ext_list)
else:
#print each_abspath
if len(each_abspath)>len(root_dir)+1+len(os.path.basename(each)):
filename=each_abspath[len(root_dir)+1:]
#print filename
if len(ext_list)==0:
tmp_all_file_name_list.append(filename)
else:
for each_ext in ext_list:
if(filename.split('.')[-1]==each_ext):
#print filename
tmp_all_file_name_list.append(filename)
else:
#print each
if len(ext_list)==0:
tmp_all_file_name_list.append(each)
else:
for each_ext in ext_list:
if(each.split('.')[-1]==each_ext):
#print each
tmp_all_file_name_list.append(each)
return tmp_all_file_name_list
def save2github(file_abs_path,repo_name,comment):
#将文件上传到github
#arg1:文件绝对路经
#arg2:远程仓库名
#提交的commit注释
local_resp_path="/root/"+repo_name
filename=os.path.basename(file_abs_path)
remote_resp_uri="https://github.com/3xp10it/%s.git" % repo_name
if os.path.exists(local_resp_path) is False:
os.system("mkdir %s && cd %s && git init && git pull %s && git remote add origin %s && git status" % (local_resp_path,local_resp_path,remote_resp_uri,remote_resp_uri))
if os.path.exists(local_resp_path+"/"+filename) is True:
print "warning!warning!warning! I will exit! There exists a same name script in local_resp_path(>>%s),and this script is downloaded from remote github repo,you should rename your script if you want to upload it to git:)" % local_resp_path+"/"+filename
return False
os.system("cp %s %s" % (file_abs_path,local_resp_path))
succeed=os.system("cd %s && git add . && git status && git commit -a -m '%s' && git push -u origin master" % (local_resp_path,comment))
if(succeed==0):
print "push succeed!!!"
return True
else:
print "push to git wrong,wrong,wrong,check it!!!"
return False
if os.path.exists(local_resp_path) is True and os.path.exists(local_resp_path+"/.git") is False:
if os.path.exists(local_resp_path+"/"+filename) is True:
print "warning!warning!warning! I will exit! There exists a same name script in local_resp_path(>>%s),you should rename your script if you want to upload it to git:)" % local_resp_path+"/"+filename
return False
os.system("mkdir /tmp/codetmp")
os.system("cd %s && cp -r * /tmp/codetmp/ && rm -r * && git init && git pull %s" % (local_resp_path,remote_resp_uri))
os.system("cp -r /tmp/codetmp/* %s && rm -r /tmp/codetmp" % local_resp_path)
os.system("cp %s %s" % (file_abs_path,local_resp_path))
succeed=os.system("cd %s && git add . && git status && git commit -a -m '%s' && git remote add origin %s && git push -u origin master" % (local_resp_path,comment,remote_resp_uri))
if(succeed==0):
print "push succeed!!!"
return True
else:
print "push to git wrong,wrong,wrong,check it!!!"
return False
if os.path.exists(local_resp_path) is True and os.path.exists(local_resp_path+"/.git") is True:
#如果本地local_resp_path存在,且文件夹中有.git,当local_resp_path文件夹中的文件与远程github仓库中的文件不一致时,
#且远程仓库有本地仓库没有的文件,选择合并本地和远程仓库并入远程仓库,所以这里采用一并重新合并的处理方法,
#(与上一个if中的情况相比,多了一个合并前先删除本地仓库中的.git文件夹的动作),
#虽然当远程仓库中不含本地仓库没有的文件时,不用这么做,但是这样做也可以处理那种情况
if os.path.exists(local_resp_path+"/"+filename) is True:
print "warning!warning!warning! I will exit! There exists a same name script in local_resp_path(>>%s),you should rename your script if you want to upload it to git:)" % local_resp_path+"/"+filename
return False
os.system("cd %s && rm -r .git" % local_resp_path)
os.system("mkdir /tmp/codetmp")
os.system("cd %s && cp -r * /tmp/codetmp/ && rm -r * && git init && git pull %s" % (local_resp_path,remote_resp_uri))
os.system("cp -r /tmp/codetmp/* %s && rm -r /tmp/codetmp" % local_resp_path)
os.system("cp %s %s" % (file_abs_path,local_resp_path))
succeed=os.system("cd %s && git add . && git status && git commit -a -m '%s' && git remote add origin %s && git push -u origin master" % (local_resp_path,comment,remote_resp_uri))
if(succeed==0):
print "push succeed!!!"
return True
else:
print "push to git wrong,wrong,wrong,check it!!!"
return False
#below code are the function about tab key complete with file path
#------------------------start-of-key-complete-with-file-path--------------------------------
import os
import sys
import readline
import glob
class tabCompleter(object):
"""
A tab completer that can either complete from
the filesystem or from a list.
Partially taken from:
http://stackoverflow.com/questions/5637124/tab-completion-in-pythons-raw-input
source code:https://gist.github.com/iamatypeofwalrus/5637895
"""
def pathCompleter(self,text,state):
"""
This is the tab completer for systems paths.
Only tested on *nix systems
"""
line = readline.get_line_buffer().split()
return [x for x in glob.glob(text+'*')][state]
def createListCompleter(self,ll):
"""
This is a closure that creates a method that autocompletes from
the given list.
Since the autocomplete function can't be given a list to complete from
a closure is used to create the listCompleter function with a list to complete
from.
"""
def listCompleter(text,state):
line = readline.get_line_buffer()
if not line:
return [c + " " for c in ll][state]
else:
return [c + " " for c in ll if c.startswith(line)][state]
self.listCompleter = listCompleter
t = tabCompleter()
t.createListCompleter(["ab","aa","bcd","bdf"])
readline.set_completer_delims('\t')
readline.parse_and_bind("tab: complete")
#readline.set_completer(t.listCompleter)
#ans = raw_input("Complete from list ")
#print ans
readline.set_completer(t.pathCompleter)
#--------------------------end-of-key-complete-with-file-path--------------------------------
def get_title_from_uri(uri):
#得到uri对应的title
import mechanize
import cookielib
br = mechanize.Browser()
br.set_cookiejar(cookielib.LWPCookieJar()) # Cookie jar
br.set_handle_equiv(True) # Browser Option
br.set_handle_gzip(True)
br.set_handle_redirect(True)
br.set_handle_referer(True)
br.set_handle_robots(False)
br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
br.open(uri)
#print br.title()
return br.title()
def get_uris_from_file(file):
#从文件中获取所有uri
f=open(file,"r+")
content=f.read()
#print content
f.close()
allurls=[]
all=re.findall('(http(\S)+)',content,re.I)
for each in all:
allurls.append(each[0])
#print allurls
return allurls
def get_title_from_file(file):
#等到文件中的所有uri对应的title
target_allurls=get_uris_from_file(file)
print "a output file:/tmp/result.txt"
writed_urls=[]
for each in target_allurls:
f=open("/tmp/result.txt","a+")
tmp=urlparse(each)
http_domain=tmp.scheme+'://'+tmp.hostname
title=get_title_from_uri(http_domain)
time.sleep(1)
try:
if http_domain not in writed_urls:
each_line_to_write=http_domain+'\r\n'+'upon uri is:'+title+'\r\n'
print each_line_to_write
f.write(each_line_to_write)
writed_urls.append(http_domain)
except:
pass
f.close()
0x03 模块发布
1>pypi register
2>eg.
mkdir /root/mypypi
cd /root/mypypi
python setup.py register
vi exp10it.py
vi setup.py
--------------setup.py-------------
import codecs
import os
import sys
try:
from setuptools import setup
except:
from distutils.core import setup
"""
打包的用的setup必须引入,
"""
def read(fname):
"""
定义一个read方法,用来读取目录下的长描述
我们一般是将README文件中的内容读取出来作为长描述,这个会在PyPI中你这个包的页面上展现出来,
你也可以不用这个方法,自己手动写内容即可,
PyPI上支持.rst格式的文件.暂不支持.md格式的文件,<BR>.rst文件PyPI会自动把它转为HTML形式显示在你包的信息页面上.
"""
return codecs.open(os.path.join(os.path.dirname(__file__), fname)).read()
NAME = "exp10it"
PACKAGES = ['cms_identify','dicts','exps']
DESCRIPTION = "this is a package about web security"
LONG_DESCRIPTION = read("README.rst")
KEYWORDS = "web security package"
AUTHOR = "quanyechavshuo"
AUTHOR_EMAIL = "quanyechavshuo@gmail.com"
URL = "http://3xp10it.cc"
VERSION = "2.3.44"
LICENSE = "MIT"
setup(
name = NAME,
version = VERSION,
description = DESCRIPTION,
long_description = LONG_DESCRIPTION,
classifiers = [
'License :: OSI Approved :: MIT License',
'Programming Language :: Python',
'Intended Audience :: Developers',
'Operating System :: OS Independent',
],
keywords = KEYWORDS,
author = AUTHOR,
author_email = AUTHOR_EMAIL,
url = URL,
license = LICENSE,
packages = PACKAGES,
include_package_data=True,
zip_safe=True,
py_modules=['exp10it','updateapi'],
)
----------------end----------------
update时只需:
cd /root/mypypi
vi exp10it.py
vi setup.py
set version to a higher one
python setup.py sdist upload
(upload时会有warning提示没有找到__init__.py文件等,经测试通过pip install exp10it可完美利用,也即此warning可忽略)
0x04 Attention
1>python打包模块上传时,打包其他文件夹或文件:
新建MANIFEST.in文件,eg.内容如下[graft是包含文件夹]
include *.md design.jpg
graft cms_identify
graft dicts
graft exps
但是这样包含的文件夹虽然会上传到pypi中,但是pip install moduleName时在MANIFEST.in中的文件夹不会被安装到系统
dist-packages中,如果需要在pip install命令后将文件夹也安装到dist-packages中则需要在setup.py中的packages参数
中填写上面的三个文件夹名
或者如下解决方案:
https://stackoverflow.com/questions/7522250/how-to-include-package-data-with-setuptools-distribute
在manifest.in文件中写成如下内容:
include *.md design.jpg
recursive-include cms_identify *
recursive-include dicts *
recursive-include exps *
recursive-include tools *
2>本地下载安装已上传到pypi中的最新模块:
pip3 install exp10it -U --no-cache
pip install 无法安装最新python包的时候:pip3 install exp10it -U --no-cache-dir
3>换了一台机器重新上传模块到pypi时会由于没有用户名密码而错误,解决方法:
https://my.oschina.net/letiantian/blog/788056