Reduction of moose.lua sizing working now!

This commit is contained in:
FlightControl_Master
2017-09-26 18:47:33 +02:00
parent 11067d4bfd
commit 5558c26db7
160 changed files with 36080 additions and 229 deletions

Binary file not shown.

View File

@@ -0,0 +1,178 @@
#!/usr/bin/lua
--------------------------------------------------------------------------------
-- Copyright (c) 2012-2014 Sierra Wireless.
-- All rights reserved. This program and the accompanying materials
-- are made available under the terms of the Eclipse Public License v1.0
-- which accompanies this distribution, and is available at
-- http://www.eclipse.org/legal/epl-v10.html
--
-- Contributors:
-- Kevin KIN-FOO <kkinfoo@sierrawireless.com>
-- - initial API and implementation and initial documentation
--------------------------------------------------------------------------------
-- Check interpreter version
if _VERSION ~= "Lua 5.1" then
print("Luadocumentor is only compatible with Lua 5.1")
return
end
--
-- Defining help message.
--
-- This message is compliant to 'lapp', which will match options and arguments
-- from command line.
local help = [[luadocumentor v0.1.4: tool for Lua Documentation Language
-f, --format (default doc) Define output format :
* doc: Will produce HTML documentation from specified file(s) or directories.
* api: Will produce API file(s) from specified file(s) or directories.
-d, --dir (default docs) Define an output directory. If the given directory doesn't exist, it will be created.
-h, --help Display the help.
-n, --noheuristic Do not use code analysis, use only comments to generate documentation.
-s, --style (default !) The path of your own css file, if you don't want to use the default one. (usefull only for the doc format)
[directories|files] Define the paths or the directories of inputs files. Only Lua or C files containing a @module tag will be considered.
]]
local docgenerator = require 'docgenerator'
local lddextractor = require 'lddextractor'
local lapp = require 'pl.lapp'
local args = lapp( help )
if not args or #args < 1 then
print('No directory provided')
return
elseif args.help then
-- Just print help
print( help )
return
end
--
-- define css file name
--
local cssfilename = "stylesheet.css"
--
-- Parse files from given folders
--
-- Check if all folders exist
local fs = require 'fs.lfs'
local allpresent, missing = fs.checkdirectory(args)
-- Some of given directories are absent
if missing then
-- List missing directories
print 'Unable to open'
for _, file in ipairs( missing ) do
print('\t'.. file)
end
return
end
-- Get files from given directories
local filestoparse, error = fs.filelist( args )
if not filestoparse then
print ( error )
return
end
--
-- Generate documentation only files
--
if args.format == 'api' then
for _, filename in ipairs( filestoparse ) do
-- Loading file content
print('Dealing with "'..filename..'".')
local file, error = io.open(filename, 'r')
if not file then
print ('Unable to open "'..filename.."'.\n"..error)
else
local code = file:read('*all')
file:close()
--
-- Creating comment file
--
local commentfile, error = lddextractor.generatecommentfile(filename, code)
-- Getting module name
-- Optimize me
local module, moduleerror = lddextractor.generateapimodule(filename, code)
if not commentfile then
print('Unable to create documentation file for "'..filename..'"\n'..error)
elseif not module or not module.name then
local error = moduleerror and '\n'..moduleerror or ''
print('Unable to compute module name for "'..filename..'".'..error)
else
--
-- Flush documentation file on disk
--
local path = args.dir..fs.separator..module.name..'.lua'
local status, err = fs.fill(path, commentfile)
if not status then
print(err)
end
end
end
end
print('Done')
return
end
-- Deal only supported output types
if args.format ~= 'doc' then
print ('"'..args.format..'" format is not handled.')
return
end
-- Generate html form files
local parsedfiles, unparsed = docgenerator.generatedocforfiles(filestoparse, cssfilename,args.noheuristic)
-- Show warnings on unparsed files
if #unparsed > 0 then
for _, faultyfile in ipairs( unparsed ) do
print( faultyfile )
end
end
-- This loop is just for counting parsed files
-- TODO: Find a more elegant way to do it
local parsedfilescount = 0
for _, p in pairs(parsedfiles) do
parsedfilescount = parsedfilescount + 1
end
print (parsedfilescount .. ' file(s) parsed.')
-- Create html files
local generated = 0
for _, apifile in pairs ( parsedfiles ) do
local status, err = fs.fill(args.dir..fs.separator..apifile.name..'.html', apifile.body)
if status then
generated = generated + 1
else
print( 'Unable to create '..apifile.name..'.html on disk.')
end
end
print (generated .. ' file(s) generated.')
-- Copying css
local csscontent
if args.style == '!' then
csscontent = require 'defaultcss'
else
local css, error = io.open(args.style, 'r')
if not css then
print('Unable to open "'..args.style .. '".\n'..error)
return
end
csscontent = css:read("*all")
css:close()
end
local status, error = fs.fill(args.dir..fs.separator..cssfilename, csscontent)
if not status then
print(error)
return
end
print('Adding css')
print('Done')

View File

@@ -0,0 +1,198 @@
Eclipse Public License - v 1.0
THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC
LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM
CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT.
1. DEFINITIONS
"Contribution" means:
a) in the case of the initial Contributor, the initial code and documentation
distributed under this Agreement, and
b) in the case of each subsequent Contributor:
i) changes to the Program, and
ii) additions to the Program;
where such changes and/or additions to the Program originate from and are
distributed by that particular Contributor. A Contribution 'originates' from
a Contributor if it was added to the Program by such Contributor itself or
anyone acting on such Contributor's behalf. Contributions do not include
additions to the Program which: (i) are separate modules of software
distributed in conjunction with the Program under their own license
agreement, and (ii) are not derivative works of the Program.
"Contributor" means any person or entity that distributes the Program.
"Licensed Patents" mean patent claims licensable by a Contributor which are
necessarily infringed by the use or sale of its Contribution alone or when
combined with the Program.
"Program" means the Contributions distributed in accordance with this Agreement.
"Recipient" means anyone who receives the Program under this Agreement,
including all Contributors.
2. GRANT OF RIGHTS
a) Subject to the terms of this Agreement, each Contributor hereby grants
Recipient a non-exclusive, worldwide, royalty-free copyright license to
reproduce, prepare derivative works of, publicly display, publicly perform,
distribute and sublicense the Contribution of such Contributor, if any, and
such derivative works, in source code and object code form.
b) Subject to the terms of this Agreement, each Contributor hereby grants
Recipient a non-exclusive, worldwide, royalty-free patent license under
Licensed Patents to make, use, sell, offer to sell, import and otherwise
transfer the Contribution of such Contributor, if any, in source code and
object code form. This patent license shall apply to the combination of the
Contribution and the Program if, at the time the Contribution is added by
the Contributor, such addition of the Contribution causes such combination
to be covered by the Licensed Patents. The patent license shall not apply
to any other combinations which include the Contribution. No hardware per
se is licensed hereunder.
c) Recipient understands that although each Contributor grants the licenses to
its Contributions set forth herein, no assurances are provided by any
Contributor that the Program does not infringe the patent or other
intellectual property rights of any other entity. Each Contributor
disclaims any liability to Recipient for claims brought by any other entity
based on infringement of intellectual property rights or otherwise. As a
condition to exercising the rights and licenses granted hereunder, each
Recipient hereby assumes sole responsibility to secure any other
intellectual property rights needed, if any. For example, if a third party
patent license is required to allow Recipient to distribute the Program, it
is Recipient's responsibility to acquire that license before distributing
the Program.
d) Each Contributor represents that to its knowledge it has sufficient
copyright rights in its Contribution, if any, to grant the copyright
license set forth in this Agreement.
3. REQUIREMENTS
A Contributor may choose to distribute the Program in object code form under its
own license agreement, provided that:
a) it complies with the terms and conditions of this Agreement; and
b) its license agreement:
i) effectively disclaims on behalf of all Contributors all warranties and
conditions, express and implied, including warranties or conditions of
title and non-infringement, and implied warranties or conditions of
merchantability and fitness for a particular purpose;
ii) effectively excludes on behalf of all Contributors all liability for
damages, including direct, indirect, special, incidental and
consequential damages, such as lost profits;
iii) states that any provisions which differ from this Agreement are offered
by that Contributor alone and not by any other party; and
iv) states that source code for the Program is available from such
Contributor, and informs licensees how to obtain it in a reasonable
manner on or through a medium customarily used for software exchange.
When the Program is made available in source code form:
a) it must be made available under this Agreement; and
b) a copy of this Agreement must be included with each copy of the Program.
Contributors may not remove or alter any copyright notices contained within
the Program.
Each Contributor must identify itself as the originator of its Contribution, if
any, in a manner that reasonably allows subsequent Recipients to identify the
originator of the Contribution.
4. COMMERCIAL DISTRIBUTION
Commercial distributors of software may accept certain responsibilities with
respect to end users, business partners and the like. While this license is
intended to facilitate the commercial use of the Program, the Contributor who
includes the Program in a commercial product offering should do so in a manner
which does not create potential liability for other Contributors. Therefore, if
a Contributor includes the Program in a commercial product offering, such
Contributor ("Commercial Contributor") hereby agrees to defend and indemnify
every other Contributor ("Indemnified Contributor") against any losses, damages
and costs (collectively "Losses") arising from claims, lawsuits and other legal
actions brought by a third party against the Indemnified Contributor to the
extent caused by the acts or omissions of such Commercial Contributor in
connection with its distribution of the Program in a commercial product
offering. The obligations in this section do not apply to any claims or Losses
relating to any actual or alleged intellectual property infringement. In order
to qualify, an Indemnified Contributor must: a) promptly notify the Commercial
Contributor in writing of such claim, and b) allow the Commercial Contributor to
control, and cooperate with the Commercial Contributor in, the defense and any
related settlement negotiations. The Indemnified Contributor may participate in
any such claim at its own expense.
For example, a Contributor might include the Program in a commercial product
offering, Product X. That Contributor is then a Commercial Contributor. If that
Commercial Contributor then makes performance claims, or offers warranties
related to Product X, those performance claims and warranties are such
Commercial Contributor's responsibility alone. Under this section, the
Commercial Contributor would have to defend claims against the other
Contributors related to those performance claims and warranties, and if a court
requires any other Contributor to pay any damages as a result, the Commercial
Contributor must pay those damages.
5. NO WARRANTY
EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR
IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE,
NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each
Recipient is solely responsible for determining the appropriateness of using and
distributing the Program and assumes all risks associated with its exercise of
rights under this Agreement , including but not limited to the risks and costs
of program errors, compliance with applicable laws, damage to or loss of data,
programs or equipment, and unavailability or interruption of operations.
6. DISCLAIMER OF LIABILITY
EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY
CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST
PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS
GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
7. GENERAL
If any provision of this Agreement is invalid or unenforceable under applicable
law, it shall not affect the validity or enforceability of the remainder of the
terms of this Agreement, and without further action by the parties hereto, such
provision shall be reformed to the minimum extent necessary to make such
provision valid and enforceable.
If Recipient institutes patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Program itself
(excluding combinations of the Program with other software or hardware)
infringes such Recipient's patent(s), then such Recipient's rights granted under
Section 2(b) shall terminate as of the date such litigation is filed.
All Recipient's rights under this Agreement shall terminate if it fails to
comply with any of the material terms or conditions of this Agreement and does
not cure such failure in a reasonable period of time after becoming aware of
such noncompliance. If all Recipient's rights under this Agreement terminate,
Recipient agrees to cease use and distribution of the Program as soon as
reasonably practicable. However, Recipient's obligations under this Agreement
and any licenses granted by Recipient relating to the Program shall continue and
survive.
Everyone is permitted to copy and distribute copies of this Agreement, but in
order to avoid inconsistency the Agreement is copyrighted and may only be
modified in the following manner. The Agreement Steward reserves the right to
publish new versions (including revisions) of this Agreement from time to time.
No one other than the Agreement Steward has the right to modify this Agreement.
The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation
may assign the responsibility to serve as the Agreement Steward to a suitable
separate entity. Each new version of the Agreement will be given a
distinguishing version number. The Program (including Contributions) may always
be distributed subject to the version of the Agreement under which it was
received. In addition, after a new version of the Agreement is published,
Contributor may elect to distribute the Program (including its Contributions)
under the new version. Except as expressly stated in Sections 2(a) and 2(b)
above, Recipient receives no rights or licenses to the intellectual property of
any Contributor under this Agreement, whether expressly, by implication,
estoppel or otherwise. All rights in the Program not expressly granted under
this Agreement are reserved.
This Agreement is governed by the laws of the State of New York and the
intellectual property laws of the United States of America. No party to this
Agreement will bring a legal action under this Agreement more than one year
after the cause of action arose. Each party waives its rights to a jury trial in
any resulting litigation.

View File

@@ -0,0 +1,7 @@
# Lua Documentor
LuaDocumentor allow users to generate HTML and API files from code documented
using Lua documentation language.
Documentation is
[available here](http://wiki.eclipse.org/Koneki/LDT/User_Area/LuaDocumentor).

View File

@@ -0,0 +1,57 @@
package = 'LuaDocumentor'
version = '0.1.5-1'
description = {
summary = 'LuaDocumentor allow users to generate HTML and API files from code documented using Lua documentation language.',
detailed = [[
This is an example for the LuaRocks tutorial.
Here we would put a detailed, typically
paragraph-long description.
]],
homepage = 'http://wiki.eclipse.org/Koneki/LDT/User_Area/LuaDocumentor',
license = 'EPL'
}
source = {
url = 'git://github.com/LuaDevelopmentTools/luadocumentor.git',
tag = 'v0.1.5-1'
}
dependencies = {
'lua ~> 5.1',
'luafilesystem ~> 1.6',
'markdown ~> 0.32',
'metalua-compiler ~> 0.7',
'penlight ~> 0.9'
}
build = {
type = 'builtin',
install = {
bin = {
luadocumentor = 'luadocumentor.lua'
},
lua = {
['models.internalmodelbuilder'] = 'models/internalmodelbuilder.mlua'
}
},
modules = {
defaultcss = 'defaultcss.lua',
docgenerator = 'docgenerator.lua',
extractors = 'extractors.lua',
lddextractor = 'lddextractor.lua',
templateengine = 'templateengine.lua',
['fs.lfs'] = 'fs/lfs.lua',
['models.apimodel'] = 'models/apimodel.lua',
['models.apimodelbuilder'] = 'models/apimodelbuilder.lua',
['models.internalmodel'] = 'models/internalmodel.lua',
['models.ldparser'] = 'models/ldparser.lua',
['template.file'] = 'template/file.lua',
['template.index'] = 'template/index.lua',
['template.index.recordtypedef'] = 'template/index/recordtypedef.lua',
['template.item'] = 'template/item.lua',
['template.page'] = 'template/page.lua',
['template.recordtypedef'] = 'template/recordtypedef.lua',
['template.usage'] = 'template/usage.lua',
['template.utils'] = 'template/utils.lua',
}
}

View File

@@ -0,0 +1,39 @@
rock_manifest = {
bin = {
luadocumentor = "bc5cc07f56db2cf1dbe80f0827332873"
},
doc = {
LICENSE = "52a21f73ac77fd790dc40dc5acda0fc2",
["README.md"] = "fcef1f43c69f3559b347d854b2626deb"
},
lua = {
["defaultcss.lua"] = "dd9b2b89e5080972bbb52056247c0c65",
["docgenerator.lua"] = "92d0a3947d88226340014d2f033be37f",
["extractors.lua"] = "74191695e5217706ee355925e5ca40fa",
fs = {
["lfs.lua"] = "4d00f9bc942b02a86ccea16544d3e85d"
},
["lddextractor.lua"] = "56edde775a5d57818aa0a07b4f723536",
models = {
["apimodel.lua"] = "3c401de18691b1222b0ad253958260ee",
["apimodelbuilder.lua"] = "4c4a3c0b48b404973542dd99f994eb2c",
["internalmodel.lua"] = "a1a21e50af8db0f0a0b9d164ccc08853",
["internalmodelbuilder.mlua"] = "ff95dfca573ccc1c19a79434e96a492d",
["ldparser.lua"] = "538904a3adbfff4ff83deda029847323"
},
template = {
["file.lua"] = "41f095bc049ef161060d8e3b4ac9de63",
index = {
["recordtypedef.lua"] = "0977ff0048a837389c2ac10285eb1ce1"
},
["index.lua"] = "5a3b3cface3b1fd9cb2d56f1edd5487b",
["item.lua"] = "5d5a6d9bffd8935c4ed283105ede331b",
["page.lua"] = "351f4a7215272f7e448faeece4945bc0",
["recordtypedef.lua"] = "69938e1d60e94eed7f95b0999f1386ca",
["usage.lua"] = "979503deb84877cb221130a5be7c1535",
["utils.lua"] = "ad97fb4e3de9fb6480b25cdd877b50d9"
},
["templateengine.lua"] = "09bfc6350e14f4ab509d14fb0fb295c0"
},
["luadocumentor-0.1.5-1.rockspec"] = "4ba1b88898dce89e7fd8fb6a700496a4"
}

View File

@@ -0,0 +1,212 @@
body {
margin-left: 1em;
margin-right: 1em;
font-family: arial, helvetica, geneva, sans-serif;
background-color:#ffffff; margin:0px;
}
code {
font-family: "Andale Mono", monospace;
}
tt {
font-family: "Andale Mono", monospace;
}
body, td, th { font-size: 11pt; }
h1, h2, h3, h4 { margin-left: 0em; }
textarea, pre, tt { font-size:10pt; }
body, td, th { color:#000000; }
small { font-size:0.85em; }
h1 { font-size:1.5em; }
h2 { font-size:1.25em; }
h3 { font-size:1.15em; }
h4 { font-size:1.06em; }
a:link { font-weight:bold; color: #004080; text-decoration: none; }
a:visited { font-weight:bold; color: #006699; text-decoration: none; }
a:link:hover { text-decoration:underline; }
hr { color:#cccccc }
img { border-width: 0px; }
h3 { padding-top: 1em; }
p { margin-left: 1em; }
p.name {
font-family: "Andale Mono", monospace;
padding-top: 1em;
margin-left: 0em;
}
blockquote { margin-left: 3em; }
.example {
background-color: rgb(245, 245, 245);
border-top-width: 1px;
border-right-width: 1px;
border-bottom-width: 1px;
border-left-width: 1px;
border-top-style: solid;
border-right-style: solid;
border-bottom-style: solid;
border-left-style: solid;
border-top-color: silver;
border-right-color: silver;
border-bottom-color: silver;
border-left-color: silver;
padding: 1em;
margin-left: 1em;
margin-right: 1em;
font-family: "Andale Mono", monospace;
font-size: smaller;
}
hr {
margin-left: 0em;
background: #00007f;
border: 0px;
height: 1px;
}
ul { list-style-type: disc; }
table.index { border: 1px #00007f; }
table.index td { text-align: left; vertical-align: top; }
table.index ul { padding-top: 0em; margin-top: 0em; }
table {
border: 1px solid black;
border-collapse: collapse;
margin-left: auto;
margin-right: auto;
}
th {
border: 1px solid black;
padding: 0.5em;
}
td {
border: 1px solid black;
padding: 0.5em;
}
div.header, div.footer { margin-left: 0em; }
#container {
margin-left: 1em;
margin-right: 1em;
background-color: #f0f0f0;
}
#product {
text-align: center;
border-bottom: 1px solid #cccccc;
background-color: #ffffff;
}
#product big {
font-size: 2em;
}
#product_logo {
}
#product_name {
}
#product_description {
}
#main {
background-color: #f0f0f0;
border-left: 2px solid #cccccc;
}
#navigation {
float: left;
width: 12em;
margin: 0;
vertical-align: top;
background-color: #f0f0f0;
overflow:visible;
}
#navigation h1 {
background-color:#e7e7e7;
font-size:1.1em;
color:#000000;
text-align:left;
margin:0px;
padding:0.2em;
border-top:1px solid #dddddd;
border-bottom:1px solid #dddddd;
}
#navigation ul {
font-size:1em;
list-style-type: none;
padding: 0;
margin: 1px;
}
#navigation li {
text-indent: -1em;
margin: 0em 0em 0em 0.5em;
display: block;
padding: 3px 0px 0px 12px;
}
#navigation li li a {
padding: 0px 3px 0px -1em;
}
#content {
margin-left: 12em;
padding: 1em;
border-left: 2px solid #cccccc;
border-right: 2px solid #cccccc;
background-color: #ffffff;
}
#about {
clear: both;
margin: 0;
padding: 5px;
border-top: 2px solid #cccccc;
background-color: #ffffff;
}
@media print {
body {
font: 10pt "Times New Roman", "TimeNR", Times, serif;
}
a {
font-weight:bold; color: #004080; text-decoration: underline;
}
#main {
background-color: #ffffff; border-left: 0px;
}
#container {
margin-left: 2%; margin-right: 2%; background-color: #ffffff;
}
#content {
margin-left: 0px; padding: 1em; border-left: 0px; border-right: 0px; background-color: #ffffff;
}
#navigation {
display: none;
}
#product_logo {
display: none;
}
#about img {
display: none;
}
.example {
font-family: "Andale Mono", monospace;
font-size: 8pt;
page-break-inside: avoid;
}
}

View File

@@ -0,0 +1,103 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<title>LuaFileSystem</title>
<link rel="stylesheet" href="doc.css" type="text/css"/>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
</head>
<body>
<div id="container">
<div id="product">
<div id="product_logo">
<a href="http://www.keplerproject.org">
<img alt="LuaFileSystem" src="luafilesystem.png"/>
</a>
</div>
<div id="product_name"><big><strong>LuaFileSystem</strong></big></div>
<div id="product_description">File System Library for the Lua Programming Language</div>
</div> <!-- id="product" -->
<div id="main">
<div id="navigation">
<h1>LuaFileSystem</h1>
<ul>
<li><a href="index.html">Home</a>
<ul>
<li><a href="index.html#overview">Overview</a></li>
<li><a href="index.html#status">Status</a></li>
<li><a href="index.html#download">Download</a></li>
<li><a href="index.html#history">History</a></li>
<li><a href="index.html#credits">Credits</a></li>
<li><a href="index.html#contact">Contact us</a></li>
</ul>
</li>
<li><a href="manual.html">Manual</a>
<ul>
<li><a href="manual.html#introduction">Introduction</a></li>
<li><a href="manual.html#building">Building</a></li>
<li><a href="manual.html#installation">Installation</a></li>
<li><a href="manual.html#reference">Reference</a></li>
</ul>
</li>
<li><strong>Examples</strong></li>
<li><a href="https://github.com/keplerproject/luafilesystem">Project</a>
<ul>
<li><a href="https://github.com/keplerproject/luafilesystem/issues">Bug Tracker</a></li>
<li><a href="https://github.com/keplerproject/luafilesystem">Git</a></li>
</ul>
</li>
<li><a href="license.html">License</a></li>
</ul>
</div> <!-- id="navigation" -->
<div id="content">
<h2><a name="example"></a>Examples</h2>
<h3>Directory iterator</h3>
<p>The following example iterates over a directory and recursively lists the
attributes for each file inside it.</p>
<pre class="example">
local lfs = require"lfs"
function attrdir (path)
for file in lfs.dir(path) do
if file ~= "." and file ~= ".." then
local f = path..'/'..file
print ("\t "..f)
local attr = lfs.attributes (f)
assert (type(attr) == "table")
if attr.mode == "directory" then
attrdir (f)
else
for name, value in pairs(attr) do
print (name, value)
end
end
end
end
end
attrdir (".")
</pre>
</div> <!-- id="content" -->
</div> <!-- id="main" -->
<div id="about">
<p><a href="http://validator.w3.org/check?uri=referer">Valid XHTML 1.0!</a></p>
<p><small>$Id: examples.html,v 1.8 2007/12/14 15:28:04 carregal Exp $</small></p>
</div> <!-- id="about" -->
</div> <!-- id="container" -->
</body>
</html>

View File

@@ -0,0 +1,218 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<title>LuaFileSystem</title>
<link rel="stylesheet" href="doc.css" type="text/css"/>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
</head>
<body>
<div id="container">
<div id="product">
<div id="product_logo">
<a href="http://www.keplerproject.org">
<img alt="LuaFileSystem" src="luafilesystem.png"/>
</a>
</div>
<div id="product_name"><big><strong>LuaFileSystem</strong></big></div>
<div id="product_description">File System Library for the Lua Programming Language</div>
</div> <!-- id="product" -->
<div id="main">
<div id="navigation">
<h1>LuaFileSystem</h1>
<ul>
<li><strong>Home</strong>
<ul>
<li><a href="index.html#overview">Overview</a></li>
<li><a href="index.html#status">Status</a></li>
<li><a href="index.html#download">Download</a></li>
<li><a href="index.html#history">History</a></li>
<li><a href="index.html#credits">Credits</a></li>
<li><a href="index.html#contact">Contact us</a></li>
</ul>
</li>
<li><a href="manual.html">Manual</a>
<ul>
<li><a href="manual.html#introduction">Introduction</a></li>
<li><a href="manual.html#building">Building</a></li>
<li><a href="manual.html#installation">Installation</a></li>
<li><a href="manual.html#reference">Reference</a></li>
</ul>
</li>
<li><a href="examples.html">Examples</a></li>
<li><a href="https://github.com/keplerproject/luafilesystem">Project</a>
<ul>
<li><a href="https://github.com/keplerproject/luafilesystem/issues">Bug Tracker</a></li>
<li><a href="https://github.com/keplerproject/luafilesystem">Git</a></li>
</ul>
</li>
<li><a href="license.html">License</a></li>
</ul>
</div> <!-- id="navigation" -->
<div id="content">
<h2><a name="overview"></a>Overview</h2>
<p>LuaFileSystem is a <a href="http://www.lua.org">Lua</a> library
developed to complement the set of functions related to file
systems offered by the standard Lua distribution.</p>
<p>LuaFileSystem offers a portable way to access
the underlying directory structure and file attributes.</p>
<p>LuaFileSystem is free software and uses the same
<a href="license.html">license</a> as Lua 5.1.</p>
<h2><a name="status"></a>Status</h2>
<p>Current version is 1.6.3. It works with Lua 5.1, 5.2 and 5.3.</p>
<h2><a name="download"></a>Download</h2>
<p>LuaFileSystem source can be downloaded from its
<a href="http://github.com/keplerproject/luafilesystem">Github</a>
page.</p>
<h2><a name="history"></a>History</h2>
<dl class="history">
<dt><strong>Version 1.6.3</strong> [15/Jan/2015]</dt>
<dd><ul>
<li>Lua 5.3 support.</li>
<li>Assorted bugfixes.</li>
</ul></dd>
<dt><strong>Version 1.6.2</strong> [??/Oct/2012]</dt>
<dd><ul>
<li>Full Lua 5.2 compatibility (with Lua 5.1 fallbacks)</li>
</ul></dd>
<dt><strong>Version 1.6.1</strong> [01/Oct/2012]</dt>
<dd><ul>
<li>fix build for Lua 5.2</li>
</ul></dd>
<dt><strong>Version 1.6.0</strong> [26/Sep/2012]</dt>
<dd><ul>
<li>getcwd fix for Android</li>
<li>support for Lua 5.2</li>
<li>add lfs.link</li>
<li>other bug fixes</li>
</ul></dd>
<dt><strong>Version 1.5.0</strong> [20/Oct/2009]</dt>
<dd><ul>
<li>Added explicit next and close methods to second return value of lfs.dir
(the directory object), for explicit iteration or explicit closing.</li>
<li>Added directory locking via lfs.lock_dir function (see the <a href="manual.html">manual</a>).</li>
</ul></dd>
<dt><strong>Version 1.4.2</strong> [03/Feb/2009]</dt>
<dd>
<ul>
<li>fixed bug [<a href="http://luaforge.net/tracker/?func=detail&amp;group_id=66&amp;aid=13198&amp;atid=356">#13198</a>]
lfs.attributes(filename, 'size') overflow on files > 2 Gb again (bug report and patch by KUBO Takehiro).</li>
<li>fixed bug [<a href="http://luaforge.net/tracker/?group_id=66&amp;atid=356&amp;func=detail&amp;aid=39794">#39794</a>]
Compile error on Solaris 10 (bug report and patch by Aaron B).</li>
<li>fixed compilation problems with Borland C.</li>
</ul>
</dd>
<dt><strong>Version 1.4.1</strong> [07/May/2008]</dt>
<dd>
<ul>
<li>documentation review</li>
<li>fixed Windows compilation issues</li>
<li>fixed bug in the Windows tests (patch by Shmuel Zeigerman)</li>
<li>fixed bug [<a href="http://luaforge.net/tracker/?func=detail&amp;group_id=66&amp;aid=2185&amp;atid=356">#2185</a>]
<code>lfs.attributes(filename, 'size')</code> overflow on files > 2 Gb
</li>
</ul>
</dd>
<dt><strong>Version 1.4.0</strong> [13/Feb/2008]</dt>
<dd>
<ul>
<li>added function
<a href="manual.html#setmode"><code>lfs.setmode</code></a>
(works only in Windows systems).</li>
<li><a href="manual.html#attributes"><code>lfs.attributes</code></a>
raises an error if attribute does not exist</li>
</ul>
</dd>
<dt><strong>Version 1.3.0</strong> [26/Oct/2007]</dt>
<dd>
<ul>
<li>added function
<a href="manual.html#symlinkattributes"><code>lfs.symlinkattributes</code></a>
(works only in non Windows systems).</li>
</ul>
</dd>
<dt><strong>Version 1.2.1</strong> [08/May/2007]</dt>
<dd>
<ul>
<li>compatible only with Lua 5.1 (Lua 5.0 support was dropped)</li>
</ul>
</dd>
<dt><strong>Version 1.2</strong> [15/Mar/2006]</dt>
<dd>
<ul>
<li>added optional argument to
<a href="manual.html#attributes"><code>lfs.attributes</code></a></li>
<li>added function
<a href="manual.html#rmdir"><code>lfs.rmdir</code></a></li>
<li>bug correction on <a href="manual.html#dir"><code>lfs.dir</code></a></li>
</ul>
</dd>
<dt><strong>Version 1.1</strong> [30/May/2005]</dt>
<dd>
<ul>
<li>added function <a href="manual.html#touch"><code>lfs.touch</code></a>.</li>
</ul>
</dd>
<dt><strong>Version 1.0</strong> [21/Jan/2005]</dt>
<dd />
<dt><strong>Version 1.0 Beta</strong> [10/Nov/2004]</dt>
<dd />
</dl>
<h2><a name="credits"></a>Credits</h2>
<p>LuaFileSystem was designed by Roberto Ierusalimschy,
Andr&eacute; Carregal and Tom&aacute;s Guisasola as part of the
<a href="http://www.keplerproject.org">Kepler Project</a>,
which holds its copyright. LuaFileSystem is currently maintained by F&aacute;bio Mascarenhas.</p>
<h2><a name="contact"></a>Contact us</h2>
<p>For more information please
<a href="mailto:info-NO-SPAM-THANKS@keplerproject.org">contact us</a>.
Comments are welcome!</p>
<p>You can also reach other Kepler developers and users on the Kepler Project
<a href="http://luaforge.net/mail/?group_id=104">mailing list</a>.</p>
</div> <!-- id="content" -->
</div> <!-- id="main" -->
<div id="about">
<p><a href="http://validator.w3.org/check?uri=referer">Valid XHTML 1.0!</a></p>
<p><small>$Id: index.html,v 1.44 2009/02/04 21:21:33 carregal Exp $</small></p>
</div> <!-- id="about" -->
</div> <!-- id="container" -->
</body>
</html>

View File

@@ -0,0 +1,122 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<title>LuaFileSystem</title>
<link rel="stylesheet" href="doc.css" type="text/css"/>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
</head>
<body>
<div id="container">
<div id="product">
<div id="product_logo">
<a href="http://www.keplerproject.org">
<img alt="LuaFileSystem" src="luafilesystem.png"/>
</a>
</div>
<div id="product_name"><big><strong>LuaFileSystem</strong></big></div>
<div id="product_description">File System Library for the Lua Programming Language</div>
</div> <!-- id="product" -->
<div id="main">
<div id="navigation">
<h1>LuaFileSystem</h1>
<ul>
<li><a href="index.html">Home</a>
<ul>
<li><a href="index.html#overview">Overview</a></li>
<li><a href="index.html#status">Status</a></li>
<li><a href="index.html#download">Download</a></li>
<li><a href="index.html#history">History</a></li>
<li><a href="index.html#credits">Credits</a></li>
<li><a href="index.html#contact">Contact us</a></li>
</ul>
</li>
<li><a href="manual.html">Manual</a>
<ul>
<li><a href="manual.html#introduction">Introduction</a></li>
<li><a href="manual.html#building">Building</a></li>
<li><a href="manual.html#installation">Installation</a></li>
<li><a href="manual.html#reference">Reference</a></li>
</ul>
</li>
<li><a href="examples.html">Examples</a></li>
<li><a href="https://github.com/keplerproject/luafilesystem">Project</a>
<ul>
<li><a href="https://github.com/keplerproject/luafilesystem/issues/">Bug Tracker</a></li>
<li><a href="https://github.com/keplerproject/luafilesystem">Git</a></li>
</ul>
</li>
<li><strong>License</strong></li>
</ul>
</div> <!-- id="navigation" -->
<div id="content">
<h1>License</h1>
<p>
LuaFileSystem is free software: it can be used for both academic
and commercial purposes at absolutely no cost. There are no
royalties or GNU-like "copyleft" restrictions. LuaFileSystem
qualifies as
<a href="http://www.opensource.org/docs/definition.html">Open Source</a>
software.
Its licenses are compatible with
<a href="http://www.gnu.org/licenses/gpl.html">GPL</a>.
LuaFileSystem is not in the public domain and the
<a href="http://www.keplerproject.org">Kepler Project</a>
keep its copyright.
The legal details are below.
</p>
<p>The spirit of the license is that you are free to use
LuaFileSystem for any purpose at no cost without having to ask us.
The only requirement is that if you do use LuaFileSystem, then you
should give us credit by including the appropriate copyright notice
somewhere in your product or its documentation.</p>
<p>The LuaFileSystem library is designed and implemented by Roberto
Ierusalimschy, Andr&eacute; Carregal and Tom&aacute;s Guisasola.
The implementation is not derived from licensed software.</p>
<hr/>
<p>Copyright &copy; 2003 Kepler Project.</p>
<p>Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use, copy,
modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:</p>
<p>The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.</p>
<p>THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.</p>
</div> <!-- id="content" -->
</div> <!-- id="main" -->
<div id="about">
<p><a href="http://validator.w3.org/check?uri=referer">Valid XHTML 1.0!</a></p>
<p><small>$Id: license.html,v 1.13 2008/02/11 22:42:21 carregal Exp $</small></p>
</div><!-- id="about" -->
</div><!-- id="container" -->
</body>
</html>

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.3 KiB

View File

@@ -0,0 +1,280 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<title>LuaFileSystem</title>
<link rel="stylesheet" href="doc.css" type="text/css"/>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
</head>
<body>
<div id="container">
<div id="product">
<div id="product_logo">
<a href="http://www.keplerproject.org"><img alt="LuaFileSystem" src="luafilesystem.png"/></a>
</div>
<div id="product_name"><big><strong>LuaFileSystem</strong></big></div>
<div id="product_description">File System Library for the Lua Programming Language</div>
</div> <!-- id="product" -->
<div id="main">
<div id="navigation">
<h1>LuaFileSystem</h1>
<ul>
<li><a href="index.html">Home</a>
<ul>
<li><a href="index.html#overview">Overview</a></li>
<li><a href="index.html#status">Status</a></li>
<li><a href="index.html#download">Download</a></li>
<li><a href="index.html#history">History</a></li>
<li><a href="index.html#credits">Credits</a></li>
<li><a href="index.html#contact">Contact us</a></li>
</ul>
</li>
<li><strong>Manual</strong>
<ul>
<li><a href="manual.html#introduction">Introduction</a></li>
<li><a href="manual.html#building">Building</a></li>
<li><a href="manual.html#installation">Installation</a></li>
<li><a href="manual.html#reference">Reference</a></li>
</ul>
</li>
<li><a href="examples.html">Examples</a></li>
<li><a href="https://github.com/keplerproject/luafilesystem">Project</a>
<ul>
<li><a href="https://github.com/keplerproject/luafilesystem/issues">Bug Tracker</a></li>
<li><a href="https://github.com/keplerproject/luafilesystem">Git</a></li>
</ul>
</li>
<li><a href="license.html">License</a></li>
</ul>
</div> <!-- id="navigation" -->
<div id="content">
<h2><a name="introduction"></a>Introduction</h2>
<p>LuaFileSystem is a <a href="http://www.lua.org">Lua</a> library
developed to complement the set of functions related to file
systems offered by the standard Lua distribution.</p>
<p>LuaFileSystem offers a portable way to access
the underlying directory structure and file attributes.</p>
<h2><a name="building"></a>Building</h2>
<p>
LuaFileSystem should be built with Lua 5.1 so the language library
and header files for the target version must be installed properly.
</p>
<p>
LuaFileSystem offers a Makefile and a separate configuration file,
<code>config</code>,
which should be edited to suit your installation before running
<code>make</code>.
The file has some definitions like paths to the external libraries,
compiler options and the like.
</p>
<p>On Windows, the C runtime used to compile LuaFileSystem must be the same
runtime that Lua uses, or some LuaFileSystem functions will not work.</p>
<h2><a name="installation"></a>Installation</h2>
<p>The easiest way to install LuaFileSystem is to use LuaRocks:</p>
<pre class="example">
luarocks install luafilesystem
</pre>
<p>If you prefer to install LuaFileSystem manually, the compiled binary should be copied to a directory in your
<a href="http://www.lua.org/manual/5.1/manual.html#pdf-package.cpath">C path</a>.</p>
<h2><a name="reference"></a>Reference</h2>
<p>
LuaFileSystem offers the following functions:
</p>
<dl class="reference">
<dt><a name="attributes"></a><strong><code>lfs.attributes (filepath [, aname])</code></strong></dt>
<dd>Returns a table with the file attributes corresponding to
<code>filepath</code> (or <code>nil</code> followed by an error message
in case of error).
If the second optional argument is given, then only the value of the
named attribute is returned (this use is equivalent to
<code>lfs.attributes(filepath).aname</code>, but the table is not created
and only one attribute is retrieved from the O.S.).
The attributes are described as follows;
attribute <code>mode</code> is a string, all the others are numbers,
and the time related attributes use the same time reference of
<a href="http://www.lua.org/manual/5.1/manual.html#pdf-os.time"><code>os.time</code></a>:
<dl>
<dt><strong><code>dev</code></strong></dt>
<dd>on Unix systems, this represents the device that the inode resides on. On Windows systems,
represents the drive number of the disk containing the file</dd>
<dt><strong><code>ino</code></strong></dt>
<dd>on Unix systems, this represents the inode number. On Windows systems this has no meaning</dd>
<dt><strong><code>mode</code></strong></dt>
<dd>string representing the associated protection mode (the values could be
<code>file</code>, <code>directory</code>, <code>link</code>, <code>socket</code>,
<code>named pipe</code>, <code>char device</code>, <code>block device</code> or
<code>other</code>)</dd>
<dt><strong><code>nlink</code></strong></dt>
<dd>number of hard links to the file</dd>
<dt><strong><code>uid</code></strong></dt>
<dd>user-id of owner (Unix only, always 0 on Windows)</dd>
<dt><strong><code>gid</code></strong></dt>
<dd>group-id of owner (Unix only, always 0 on Windows)</dd>
<dt><strong><code>rdev</code></strong></dt>
<dd>on Unix systems, represents the device type, for special file inodes.
On Windows systems represents the same as <code>dev</code></dd>
<dt><strong><code>access</code></strong></dt>
<dd>time of last access</dd>
<dt><strong><code>modification</code></strong></dt>
<dd>time of last data modification</dd>
<dt><strong><code>change</code></strong></dt>
<dd>time of last file status change</dd>
<dt><strong><code>size</code></strong></dt>
<dd>file size, in bytes</dd>
<dt><strong><code>blocks</code></strong></dt>
<dd>block allocated for file; (Unix only)</dd>
<dt><strong><code>blksize</code></strong></dt>
<dd>optimal file system I/O blocksize; (Unix only)</dd>
</dl>
This function uses <code>stat</code> internally thus if the given
<code>filepath</code> is a symbolic link, it is followed (if it points to
another link the chain is followed recursively) and the information
is about the file it refers to.
To obtain information about the link itself, see function
<a href="#symlinkattributes">lfs.symlinkattributes</a>.
</dd>
<dt><a name="chdir"></a><strong><code>lfs.chdir (path)</code></strong></dt>
<dd>Changes the current working directory to the given
<code>path</code>.<br />
Returns <code>true</code> in case of success or <code>nil</code> plus an
error string.</dd>
<dt><a name="chdir"></a><strong><code>lfs.lock_dir(path, [seconds_stale])</code></strong></dt>
<dd>Creates a lockfile (called lockfile.lfs) in <code>path</code> if it does not
exist and returns the lock. If the lock already exists checks if
it's stale, using the second parameter (default for the second
parameter is <code>INT_MAX</code>, which in practice means the lock will never
be stale. To free the the lock call <code>lock:free()</code>. <br/>
In case of any errors it returns nil and the error message. In
particular, if the lock exists and is not stale it returns the
"File exists" message.</dd>
<dt><a name="getcwd"></a><strong><code>lfs.currentdir ()</code></strong></dt>
<dd>Returns a string with the current working directory or <code>nil</code>
plus an error string.</dd>
<dt><a name="dir"></a><strong><code>iter, dir_obj = lfs.dir (path)</code></strong></dt>
<dd>
Lua iterator over the entries of a given directory.
Each time the iterator is called with <code>dir_obj</code> it returns a directory entry's name as a string, or
<code>nil</code> if there are no more entries. You can also iterate by calling <code>dir_obj:next()</code>, and
explicitly close the directory before the iteration finished with <code>dir_obj:close()</code>.
Raises an error if <code>path</code> is not a directory.
</dd>
<dt><a name="lock"></a><strong><code>lfs.lock (filehandle, mode[, start[, length]])</code></strong></dt>
<dd>Locks a file or a part of it. This function works on <em>open files</em>; the
file handle should be specified as the first argument.
The string <code>mode</code> could be either
<code>r</code> (for a read/shared lock) or <code>w</code> (for a
write/exclusive lock). The optional arguments <code>start</code>
and <code>length</code> can be used to specify a starting point and
its length; both should be numbers.<br />
Returns <code>true</code> if the operation was successful; in
case of error, it returns <code>nil</code> plus an error string.
</dd>
<dt><a name="link"></a><strong><code>lfs.link (old, new[, symlink])</code></strong></dt>
<dd>Creates a link. The first argument is the object to link to
and the second is the name of the link. If the optional third
argument is true, the link will by a symbolic link (by default, a
hard link is created).
</dd>
<dt><a name="mkdir"></a><strong><code>lfs.mkdir (dirname)</code></strong></dt>
<dd>Creates a new directory. The argument is the name of the new
directory.<br />
Returns <code>true</code> if the operation was successful;
in case of error, it returns <code>nil</code> plus an error string.
</dd>
<dt><a name="rmdir"></a><strong><code>lfs.rmdir (dirname)</code></strong></dt>
<dd>Removes an existing directory. The argument is the name of the directory.<br />
Returns <code>true</code> if the operation was successful;
in case of error, it returns <code>nil</code> plus an error string.</dd>
<dt><a name="setmode"></a><strong><code>lfs.setmode (file, mode)</code></strong></dt>
<dd>Sets the writing mode for a file. The mode string can be either <code>"binary"</code> or <code>"text"</code>.
Returns <code>true</code> followed the previous mode string for the file, or
<code>nil</code> followed by an error string in case of errors.
On non-Windows platforms, where the two modes are identical,
setting the mode has no effect, and the mode is always returned as <code>binary</code>.
</dd>
<dt><a name="symlinkattributes"></a><strong><code>lfs.symlinkattributes (filepath [, aname])</code></strong></dt>
<dd>Identical to <a href="#attributes">lfs.attributes</a> except that
it obtains information about the link itself (not the file it refers to).
On Windows this function does not yet support links, and is identical to
<code>lfs.attributes</code>.
</dd>
<dt><a name="touch"></a><strong><code>lfs.touch (filepath [, atime [, mtime]])</code></strong></dt>
<dd>Set access and modification times of a file. This function is
a bind to <code>utime</code> function. The first argument is the
filename, the second argument (<code>atime</code>) is the access time,
and the third argument (<code>mtime</code>) is the modification time.
Both times are provided in seconds (which should be generated with
Lua standard function <code>os.time</code>).
If the modification time is omitted, the access time provided is used;
if both times are omitted, the current time is used.<br />
Returns <code>true</code> if the operation was successful;
in case of error, it returns <code>nil</code> plus an error string.
</dd>
<dt><a name="unlock"></a><strong><code>lfs.unlock (filehandle[, start[, length]])</code></strong></dt>
<dd>Unlocks a file or a part of it. This function works on
<em>open files</em>; the file handle should be specified as the first
argument. The optional arguments <code>start</code> and
<code>length</code> can be used to specify a starting point and its
length; both should be numbers.<br />
Returns <code>true</code> if the operation was successful;
in case of error, it returns <code>nil</code> plus an error string.
</dd>
</dl>
</div> <!-- id="content" -->
</div> <!-- id="main" -->
<div id="about">
<p><a href="http://validator.w3.org/check?uri=referer">Valid XHTML 1.0!</a></p>
<p><small>$Id: manual.html,v 1.45 2009/06/03 20:53:55 mascarenhas Exp $</small></p>
</div> <!-- id="about" -->
</div> <!-- id="container" -->
</body>
</html>

View File

@@ -0,0 +1,29 @@
package = "LuaFileSystem"
version = "1.6.3-2"
source = {
url = "git://github.com/keplerproject/luafilesystem",
tag = "v_1_6_3"
}
description = {
summary = "File System Library for the Lua Programming Language",
detailed = [[
LuaFileSystem is a Lua library developed to complement the set of
functions related to file systems offered by the standard Lua
distribution. LuaFileSystem offers a portable way to access the
underlying directory structure and file attributes.
]],
homepage = "http://keplerproject.github.io/luafilesystem",
license = "MIT/X11"
}
dependencies = {
"lua >= 5.1"
}
build = {
type = "builtin",
modules = {
lfs = "src/lfs.c"
},
copy_directories = {
"doc", "tests"
}
}

View File

@@ -0,0 +1,19 @@
rock_manifest = {
doc = {
us = {
["doc.css"] = "d0a913514fb190240b3b4033d105cbc0",
["examples.html"] = "5832f72021728374cf57b621d62ce0ff",
["index.html"] = "96885bdda963939f0a363b5fa6b16b59",
["license.html"] = "e3a756835cb7c8ae277d5e513c8e32ee",
["luafilesystem.png"] = "81e923e976e99f894ea0aa8b52baff29",
["manual.html"] = "d6473799b73ce486c3ea436586cb3b34"
}
},
lib = {
["lfs.dll"] = "c0e2145e1ef2815ae5fae01454291b66"
},
["luafilesystem-1.6.3-2.rockspec"] = "eb0ef7c190516892eb8357af799eea5f",
tests = {
["test.lua"] = "7b4ddb5bdb7e0b1b1ed0150d473535c9"
}
}

View File

@@ -0,0 +1,175 @@
#!/usr/bin/env lua5.1
local tmp = "/tmp"
local sep = string.match (package.config, "[^\n]+")
local upper = ".."
local lfs = require"lfs"
print (lfs._VERSION)
io.write(".")
io.flush()
function attrdir (path)
for file in lfs.dir(path) do
if file ~= "." and file ~= ".." then
local f = path..sep..file
print ("\t=> "..f.." <=")
local attr = lfs.attributes (f)
assert (type(attr) == "table")
if attr.mode == "directory" then
attrdir (f)
else
for name, value in pairs(attr) do
print (name, value)
end
end
end
end
end
-- Checking changing directories
local current = assert (lfs.currentdir())
local reldir = string.gsub (current, "^.*%"..sep.."([^"..sep.."])$", "%1")
assert (lfs.chdir (upper), "could not change to upper directory")
assert (lfs.chdir (reldir), "could not change back to current directory")
assert (lfs.currentdir() == current, "error trying to change directories")
assert (lfs.chdir ("this couldn't be an actual directory") == nil, "could change to a non-existent directory")
io.write(".")
io.flush()
-- Changing creating and removing directories
local tmpdir = current..sep.."lfs_tmp_dir"
local tmpfile = tmpdir..sep.."tmp_file"
-- Test for existence of a previous lfs_tmp_dir
-- that may have resulted from an interrupted test execution and remove it
if lfs.chdir (tmpdir) then
assert (lfs.chdir (upper), "could not change to upper directory")
assert (os.remove (tmpfile), "could not remove file from previous test")
assert (lfs.rmdir (tmpdir), "could not remove directory from previous test")
end
io.write(".")
io.flush()
-- tries to create a directory
assert (lfs.mkdir (tmpdir), "could not make a new directory")
local attrib, errmsg = lfs.attributes (tmpdir)
if not attrib then
error ("could not get attributes of file `"..tmpdir.."':\n"..errmsg)
end
local f = io.open(tmpfile, "w")
f:close()
io.write(".")
io.flush()
-- Change access time
local testdate = os.time({ year = 2007, day = 10, month = 2, hour=0})
assert (lfs.touch (tmpfile, testdate))
local new_att = assert (lfs.attributes (tmpfile))
assert (new_att.access == testdate, "could not set access time")
assert (new_att.modification == testdate, "could not set modification time")
io.write(".")
io.flush()
-- Change access and modification time
local testdate1 = os.time({ year = 2007, day = 10, month = 2, hour=0})
local testdate2 = os.time({ year = 2007, day = 11, month = 2, hour=0})
assert (lfs.touch (tmpfile, testdate2, testdate1))
local new_att = assert (lfs.attributes (tmpfile))
assert (new_att.access == testdate2, "could not set access time")
assert (new_att.modification == testdate1, "could not set modification time")
io.write(".")
io.flush()
-- Checking link (does not work on Windows)
if lfs.link (tmpfile, "_a_link_for_test_", true) then
assert (lfs.attributes"_a_link_for_test_".mode == "file")
assert (lfs.symlinkattributes"_a_link_for_test_".mode == "link")
assert (lfs.link (tmpfile, "_a_hard_link_for_test_"))
assert (lfs.attributes (tmpfile, "nlink") == 2)
assert (os.remove"_a_link_for_test_")
assert (os.remove"_a_hard_link_for_test_")
end
io.write(".")
io.flush()
-- Checking text/binary modes (only has an effect in Windows)
local f = io.open(tmpfile, "w")
local result, mode = lfs.setmode(f, "binary")
assert(result) -- on non-Windows platforms, mode is always returned as "binary"
result, mode = lfs.setmode(f, "text")
assert(result and mode == "binary")
f:close()
io.write(".")
io.flush()
-- Restore access time to current value
assert (lfs.touch (tmpfile, attrib.access, attrib.modification))
new_att = assert (lfs.attributes (tmpfile))
assert (new_att.access == attrib.access)
assert (new_att.modification == attrib.modification)
io.write(".")
io.flush()
-- Check consistency of lfs.attributes values
local attr = lfs.attributes (tmpfile)
for key, value in pairs(attr) do
assert (value == lfs.attributes (tmpfile, key),
"lfs.attributes values not consistent")
end
-- Remove new file and directory
assert (os.remove (tmpfile), "could not remove new file")
assert (lfs.rmdir (tmpdir), "could not remove new directory")
assert (lfs.mkdir (tmpdir..sep.."lfs_tmp_dir") == nil, "could create a directory inside a non-existent one")
io.write(".")
io.flush()
-- Trying to get attributes of a non-existent file
assert (lfs.attributes ("this couldn't be an actual file") == nil, "could get attributes of a non-existent file")
assert (type(lfs.attributes (upper)) == "table", "couldn't get attributes of upper directory")
io.write(".")
io.flush()
-- Stressing directory iterator
count = 0
for i = 1, 4000 do
for file in lfs.dir (tmp) do
count = count + 1
end
end
io.write(".")
io.flush()
-- Stressing directory iterator, explicit version
count = 0
for i = 1, 4000 do
local iter, dir = lfs.dir(tmp)
local file = dir:next()
while file do
count = count + 1
file = dir:next()
end
assert(not pcall(dir.next, dir))
end
io.write(".")
io.flush()
-- directory explicit close
local iter, dir = lfs.dir(tmp)
dir:close()
assert(not pcall(dir.next, dir))
print"Ok!"

View File

@@ -0,0 +1,653 @@
#!/usr/bin/env lua
---------
-- LuaSrcDiet
--
-- Compresses Lua source code by removing unnecessary characters.
-- For Lua 5.1+ source code.
--
-- **Notes:**
--
-- * Remember to update version and date information below (MSG_TITLE).
-- * TODO: passing data tables around is a horrific mess.
-- * TODO: to implement pcall() to properly handle lexer etc. errors.
-- * TODO: need some automatic testing for a semblance of sanity.
-- * TODO: the plugin module is highly experimental and unstable.
----
local equiv = require "luasrcdiet.equiv"
local fs = require "luasrcdiet.fs"
local llex = require "luasrcdiet.llex"
local lparser = require "luasrcdiet.lparser"
local luasrcdiet = require "luasrcdiet.init"
local optlex = require "luasrcdiet.optlex"
local optparser = require "luasrcdiet.optparser"
local byte = string.byte
local concat = table.concat
local find = string.find
local fmt = string.format
local gmatch = string.gmatch
local match = string.match
local print = print
local rep = string.rep
local sub = string.sub
local plugin
local LUA_VERSION = match(_VERSION, " (5%.[123])$") or "5.1"
-- Is --opt-binequiv available for this Lua version?
local BIN_EQUIV_AVAIL = LUA_VERSION == "5.1" and not package.loaded.jit
---------------------- Messages and textual data ----------------------
local MSG_TITLE = fmt([[
LuaSrcDiet: Puts your Lua 5.1+ source code on a diet
Version %s <%s>
]], luasrcdiet._VERSION, luasrcdiet._HOMEPAGE)
local MSG_USAGE = [[
usage: luasrcdiet [options] [filenames]
example:
>luasrcdiet myscript.lua -o myscript_.lua
options:
-v, --version prints version information
-h, --help prints usage information
-o <file> specify file name to write output
-s <suffix> suffix for output files (default '_')
--keep <msg> keep block comment with <msg> inside
--plugin <module> run <module> in plugin/ directory
- stop handling arguments
(optimization levels)
--none all optimizations off (normalizes EOLs only)
--basic lexer-based optimizations only
--maximum maximize reduction of source
(informational)
--quiet process files quietly
--read-only read file and print token stats only
--dump-lexer dump raw tokens from lexer to stdout
--dump-parser dump variable tracking tables from parser
--details extra info (strings, numbers, locals)
features (to disable, insert 'no' prefix like --noopt-comments):
%s
default settings:
%s]]
-- Optimization options, for ease of switching on and off.
--
-- * Positive to enable optimization, negative (no) to disable.
-- * These options should follow --opt-* and --noopt-* style for now.
local OPTION = [[
--opt-comments,'remove comments and block comments'
--opt-whitespace,'remove whitespace excluding EOLs'
--opt-emptylines,'remove empty lines'
--opt-eols,'all above, plus remove unnecessary EOLs'
--opt-strings,'optimize strings and long strings'
--opt-numbers,'optimize numbers'
--opt-locals,'optimize local variable names'
--opt-entropy,'tries to reduce symbol entropy of locals'
--opt-srcequiv,'insist on source (lexer stream) equivalence'
--opt-binequiv,'insist on binary chunk equivalence (only for PUC Lua 5.1)'
--opt-experimental,'apply experimental optimizations'
]]
-- Preset configuration.
local DEFAULT_CONFIG = [[
--opt-comments --opt-whitespace --opt-emptylines
--opt-numbers --opt-locals
--opt-srcequiv --noopt-binequiv
]]
-- Override configurations: MUST explicitly enable/disable everything.
local BASIC_CONFIG = [[
--opt-comments --opt-whitespace --opt-emptylines
--noopt-eols --noopt-strings --noopt-numbers
--noopt-locals --noopt-entropy
--opt-srcequiv --noopt-binequiv
]]
local MAXIMUM_CONFIG = [[
--opt-comments --opt-whitespace --opt-emptylines
--opt-eols --opt-strings --opt-numbers
--opt-locals --opt-entropy
--opt-srcequiv
]] .. (BIN_EQUIV_AVAIL and ' --opt-binequiv' or ' --noopt-binequiv')
local NONE_CONFIG = [[
--noopt-comments --noopt-whitespace --noopt-emptylines
--noopt-eols --noopt-strings --noopt-numbers
--noopt-locals --noopt-entropy
--opt-srcequiv --noopt-binequiv
]]
local DEFAULT_SUFFIX = "_" -- default suffix for file renaming
local PLUGIN_SUFFIX = "luasrcdiet.plugin." -- relative location of plugins
------------- Startup and initialize option list handling -------------
--- Simple error message handler; change to error if traceback wanted.
--
-- @tparam string msg The message to print.
local function die(msg)
print("LuaSrcDiet (error): "..msg); os.exit(1)
end
--die = error--DEBUG
-- Prepare text for list of optimizations, prepare lookup table.
local MSG_OPTIONS = ""
do
local WIDTH = 24
local o = {}
for op, desc in gmatch(OPTION, "%s*([^,]+),'([^']+)'") do
local msg = " "..op
msg = msg..rep(" ", WIDTH - #msg)..desc.."\n"
MSG_OPTIONS = MSG_OPTIONS..msg
o[op] = true
o["--no"..sub(op, 3)] = true
end
OPTION = o -- replace OPTION with lookup table
end
MSG_USAGE = fmt(MSG_USAGE, MSG_OPTIONS, DEFAULT_CONFIG)
--------- Global variable initialization, option set handling ---------
local suffix = DEFAULT_SUFFIX -- file suffix
local option = {} -- program options
local stat_c, stat_l -- statistics tables
--- Sets option lookup table based on a text list of options.
--
-- Note: additional forced settings for --opt-eols is done in optlex.lua.
--
-- @tparam string CONFIG
local function set_options(CONFIG)
for op in gmatch(CONFIG, "(%-%-%S+)") do
if sub(op, 3, 4) == "no" and -- handle negative options
OPTION["--"..sub(op, 5)] then
option[sub(op, 5)] = false
else
option[sub(op, 3)] = true
end
end
end
-------------------------- Support functions --------------------------
-- List of token types, parser-significant types are up to TTYPE_GRAMMAR
-- while the rest are not used by parsers; arranged for stats display.
local TTYPES = {
"TK_KEYWORD", "TK_NAME", "TK_NUMBER", -- grammar
"TK_STRING", "TK_LSTRING", "TK_OP",
"TK_EOS",
"TK_COMMENT", "TK_LCOMMENT", -- non-grammar
"TK_EOL", "TK_SPACE",
}
local TTYPE_GRAMMAR = 7
local EOLTYPES = { -- EOL names for token dump
["\n"] = "LF", ["\r"] = "CR",
["\n\r"] = "LFCR", ["\r\n"] = "CRLF",
}
--- Reads source code from the file.
--
-- @tparam string fname Path of the file to read.
-- @treturn string Content of the file.
local function load_file(fname)
local data, err = fs.read_file(fname, "rb")
if not data then die(err) end
return data
end
--- Saves source code to the file.
--
-- @tparam string fname Path of the destination file.
-- @tparam string dat The data to write into the file.
local function save_file(fname, dat)
local ok, err = fs.write_file(fname, dat, "wb")
if not ok then die(err) end
end
------------------ Functions to deal with statistics ------------------
--- Initializes the statistics table.
local function stat_init()
stat_c, stat_l = {}, {}
for i = 1, #TTYPES do
local ttype = TTYPES[i]
stat_c[ttype], stat_l[ttype] = 0, 0
end
end
--- Adds a token to the statistics table.
--
-- @tparam string tok The token.
-- @param seminfo
local function stat_add(tok, seminfo)
stat_c[tok] = stat_c[tok] + 1
stat_l[tok] = stat_l[tok] + #seminfo
end
--- Computes totals for the statistics table, returns average table.
--
-- @treturn table
local function stat_calc()
local function avg(c, l) -- safe average function
if c == 0 then return 0 end
return l / c
end
local stat_a = {}
local c, l = 0, 0
for i = 1, TTYPE_GRAMMAR do -- total grammar tokens
local ttype = TTYPES[i]
c = c + stat_c[ttype]; l = l + stat_l[ttype]
end
stat_c.TOTAL_TOK, stat_l.TOTAL_TOK = c, l
stat_a.TOTAL_TOK = avg(c, l)
c, l = 0, 0
for i = 1, #TTYPES do -- total all tokens
local ttype = TTYPES[i]
c = c + stat_c[ttype]; l = l + stat_l[ttype]
stat_a[ttype] = avg(stat_c[ttype], stat_l[ttype])
end
stat_c.TOTAL_ALL, stat_l.TOTAL_ALL = c, l
stat_a.TOTAL_ALL = avg(c, l)
return stat_a
end
----------------------------- Main tasks -----------------------------
--- A simple token dumper, minimal translation of seminfo data.
--
-- @tparam string srcfl Path of the source file.
local function dump_tokens(srcfl)
-- Load file and process source input into tokens.
local z = load_file(srcfl)
local toklist, seminfolist = llex.lex(z)
-- Display output.
for i = 1, #toklist do
local tok, seminfo = toklist[i], seminfolist[i]
if tok == "TK_OP" and byte(seminfo) < 32 then
seminfo = "("..byte(seminfo)..")"
elseif tok == "TK_EOL" then
seminfo = EOLTYPES[seminfo]
else
seminfo = "'"..seminfo.."'"
end
print(tok.." "..seminfo)
end--for
end
--- Dumps globalinfo and localinfo tables.
--
-- @tparam string srcfl Path of the source file.
local function dump_parser(srcfl)
-- Load file and process source input into tokens,
local z = load_file(srcfl)
local toklist, seminfolist, toklnlist = llex.lex(z)
-- Do parser optimization here.
local xinfo = lparser.parse(toklist, seminfolist, toklnlist)
local globalinfo, localinfo = xinfo.globalinfo, xinfo.localinfo
-- Display output.
local hl = rep("-", 72)
print("*** Local/Global Variable Tracker Tables ***")
print(hl.."\n GLOBALS\n"..hl)
-- global tables have a list of xref numbers only
for i = 1, #globalinfo do
local obj = globalinfo[i]
local msg = "("..i..") '"..obj.name.."' -> "
local xref = obj.xref
for j = 1, #xref do msg = msg..xref[j].." " end
print(msg)
end
-- Local tables have xref numbers and a few other special
-- numbers that are specially named: decl (declaration xref),
-- act (activation xref), rem (removal xref).
print(hl.."\n LOCALS (decl=declared act=activated rem=removed)\n"..hl)
for i = 1, #localinfo do
local obj = localinfo[i]
local msg = "("..i..") '"..obj.name.."' decl:"..obj.decl..
" act:"..obj.act.." rem:"..obj.rem
if obj.is_special then
msg = msg.." is_special"
end
msg = msg.." -> "
local xref = obj.xref
for j = 1, #xref do msg = msg..xref[j].." " end
print(msg)
end
print(hl.."\n")
end
--- Reads source file(s) and reports some statistics.
--
-- @tparam string srcfl Path of the source file.
local function read_only(srcfl)
-- Load file and process source input into tokens.
local z = load_file(srcfl)
local toklist, seminfolist = llex.lex(z)
print(MSG_TITLE)
print("Statistics for: "..srcfl.."\n")
-- Collect statistics.
stat_init()
for i = 1, #toklist do
local tok, seminfo = toklist[i], seminfolist[i]
stat_add(tok, seminfo)
end--for
local stat_a = stat_calc()
-- Display output.
local function figures(tt)
return stat_c[tt], stat_l[tt], stat_a[tt]
end
local tabf1, tabf2 = "%-16s%8s%8s%10s", "%-16s%8d%8d%10.2f"
local hl = rep("-", 42)
print(fmt(tabf1, "Lexical", "Input", "Input", "Input"))
print(fmt(tabf1, "Elements", "Count", "Bytes", "Average"))
print(hl)
for i = 1, #TTYPES do
local ttype = TTYPES[i]
print(fmt(tabf2, ttype, figures(ttype)))
if ttype == "TK_EOS" then print(hl) end
end
print(hl)
print(fmt(tabf2, "Total Elements", figures("TOTAL_ALL")))
print(hl)
print(fmt(tabf2, "Total Tokens", figures("TOTAL_TOK")))
print(hl.."\n")
end
--- Processes source file(s), writes output and reports some statistics.
--
-- @tparam string srcfl Path of the source file.
-- @tparam string destfl Path of the destination file where to write optimized source.
local function process_file(srcfl, destfl)
-- handle quiet option
local function print(...) --luacheck: ignore 431
if option.QUIET then return end
_G.print(...)
end
if plugin and plugin.init then -- plugin init
option.EXIT = false
plugin.init(option, srcfl, destfl)
if option.EXIT then return end
end
print(MSG_TITLE) -- title message
-- Load file and process source input into tokens.
local z = load_file(srcfl)
if plugin and plugin.post_load then -- plugin post-load
z = plugin.post_load(z) or z
if option.EXIT then return end
end
local toklist, seminfolist, toklnlist = llex.lex(z)
if plugin and plugin.post_lex then -- plugin post-lex
plugin.post_lex(toklist, seminfolist, toklnlist)
if option.EXIT then return end
end
-- Collect 'before' statistics.
stat_init()
for i = 1, #toklist do
local tok, seminfo = toklist[i], seminfolist[i]
stat_add(tok, seminfo)
end--for
local stat1_a = stat_calc()
local stat1_c, stat1_l = stat_c, stat_l
-- Do parser optimization here.
optparser.print = print -- hack
local xinfo = lparser.parse(toklist, seminfolist, toklnlist)
if plugin and plugin.post_parse then -- plugin post-parse
plugin.post_parse(xinfo.globalinfo, xinfo.localinfo)
if option.EXIT then return end
end
optparser.optimize(option, toklist, seminfolist, xinfo)
if plugin and plugin.post_optparse then -- plugin post-optparse
plugin.post_optparse()
if option.EXIT then return end
end
-- Do lexer optimization here, save output file.
local warn = optlex.warn -- use this as a general warning lookup
optlex.print = print -- hack
toklist, seminfolist, toklnlist
= optlex.optimize(option, toklist, seminfolist, toklnlist)
if plugin and plugin.post_optlex then -- plugin post-optlex
plugin.post_optlex(toklist, seminfolist, toklnlist)
if option.EXIT then return end
end
local dat = concat(seminfolist)
-- Depending on options selected, embedded EOLs in long strings and
-- long comments may not have been translated to \n, tack a warning.
if find(dat, "\r\n", 1, 1) or
find(dat, "\n\r", 1, 1) then
warn.MIXEDEOL = true
end
-- Test source and binary chunk equivalence.
equiv.init(option, llex, warn)
equiv.source(z, dat)
if BIN_EQUIV_AVAIL then
equiv.binary(z, dat)
end
local smsg = "before and after lexer streams are NOT equivalent!"
local bmsg = "before and after binary chunks are NOT equivalent!"
-- for reporting, die if option was selected, else just warn
if warn.SRC_EQUIV then
if option["opt-srcequiv"] then die(smsg) end
else
print("*** SRCEQUIV: token streams are sort of equivalent")
if option["opt-locals"] then
print("(but no identifier comparisons since --opt-locals enabled)")
end
print()
end
if warn.BIN_EQUIV then
if option["opt-binequiv"] then die(bmsg) end
elseif BIN_EQUIV_AVAIL then
print("*** BINEQUIV: binary chunks are sort of equivalent")
print()
end
-- Save optimized source stream to output file.
save_file(destfl, dat)
-- Collect 'after' statistics.
stat_init()
for i = 1, #toklist do
local tok, seminfo = toklist[i], seminfolist[i]
stat_add(tok, seminfo)
end--for
local stat_a = stat_calc()
-- Display output.
print("Statistics for: "..srcfl.." -> "..destfl.."\n")
local function figures(tt)
return stat1_c[tt], stat1_l[tt], stat1_a[tt],
stat_c[tt], stat_l[tt], stat_a[tt]
end
local tabf1, tabf2 = "%-16s%8s%8s%10s%8s%8s%10s",
"%-16s%8d%8d%10.2f%8d%8d%10.2f"
local hl = rep("-", 68)
print("*** lexer-based optimizations summary ***\n"..hl)
print(fmt(tabf1, "Lexical",
"Input", "Input", "Input",
"Output", "Output", "Output"))
print(fmt(tabf1, "Elements",
"Count", "Bytes", "Average",
"Count", "Bytes", "Average"))
print(hl)
for i = 1, #TTYPES do
local ttype = TTYPES[i]
print(fmt(tabf2, ttype, figures(ttype)))
if ttype == "TK_EOS" then print(hl) end
end
print(hl)
print(fmt(tabf2, "Total Elements", figures("TOTAL_ALL")))
print(hl)
print(fmt(tabf2, "Total Tokens", figures("TOTAL_TOK")))
print(hl)
-- Report warning flags from optimizing process.
if warn.LSTRING then
print("* WARNING: "..warn.LSTRING)
elseif warn.MIXEDEOL then
print("* WARNING: ".."output still contains some CRLF or LFCR line endings")
elseif warn.SRC_EQUIV then
print("* WARNING: "..smsg)
elseif warn.BIN_EQUIV then
print("* WARNING: "..bmsg)
end
print()
end
---------------------------- Main functions ---------------------------
local arg = {...} -- program arguments
set_options(DEFAULT_CONFIG) -- set to default options at beginning
--- Does per-file handling, ship off to tasks.
--
-- @tparam {string,...} fspec List of source files.
local function do_files(fspec)
for i = 1, #fspec do
local srcfl = fspec[i]
local destfl
-- Find and replace extension for filenames.
local extb, exte = find(srcfl, "%.[^%.%\\%/]*$")
local basename, extension = srcfl, ""
if extb and extb > 1 then
basename = sub(srcfl, 1, extb - 1)
extension = sub(srcfl, extb, exte)
end
destfl = basename..suffix..extension
if #fspec == 1 and option.OUTPUT_FILE then
destfl = option.OUTPUT_FILE
end
if srcfl == destfl then
die("output filename identical to input filename")
end
-- Perform requested operations.
if option.DUMP_LEXER then
dump_tokens(srcfl)
elseif option.DUMP_PARSER then
dump_parser(srcfl)
elseif option.READ_ONLY then
read_only(srcfl)
else
process_file(srcfl, destfl)
end
end--for
end
--- The main function.
local function main()
local fspec = {}
local argn, i = #arg, 1
if argn == 0 then
option.HELP = true
end
-- Handle arguments.
while i <= argn do
local o, p = arg[i], arg[i + 1]
local dash = match(o, "^%-%-?")
if dash == "-" then -- single-dash options
if o == "-h" then
option.HELP = true; break
elseif o == "-v" then
option.VERSION = true; break
elseif o == "-s" then
if not p then die("-s option needs suffix specification") end
suffix = p
i = i + 1
elseif o == "-o" then
if not p then die("-o option needs a file name") end
option.OUTPUT_FILE = p
i = i + 1
elseif o == "-" then
break -- ignore rest of args
else
die("unrecognized option "..o)
end
elseif dash == "--" then -- double-dash options
if o == "--help" then
option.HELP = true; break
elseif o == "--version" then
option.VERSION = true; break
elseif o == "--keep" then
if not p then die("--keep option needs a string to match for") end
option.KEEP = p
i = i + 1
elseif o == "--plugin" then
if not p then die("--plugin option needs a module name") end
if option.PLUGIN then die("only one plugin can be specified") end
option.PLUGIN = p
plugin = require(PLUGIN_SUFFIX..p)
i = i + 1
elseif o == "--quiet" then
option.QUIET = true
elseif o == "--read-only" then
option.READ_ONLY = true
elseif o == "--basic" then
set_options(BASIC_CONFIG)
elseif o == "--maximum" then
set_options(MAXIMUM_CONFIG)
elseif o == "--none" then
set_options(NONE_CONFIG)
elseif o == "--dump-lexer" then
option.DUMP_LEXER = true
elseif o == "--dump-parser" then
option.DUMP_PARSER = true
elseif o == "--details" then
option.DETAILS = true
elseif OPTION[o] then -- lookup optimization options
set_options(o)
else
die("unrecognized option "..o)
end
else
fspec[#fspec + 1] = o -- potential filename
end
i = i + 1
end--while
if option.HELP then
print(MSG_TITLE..MSG_USAGE); return true
elseif option.VERSION then
print(MSG_TITLE); return true
end
if option["opt-binequiv"] and not BIN_EQUIV_AVAIL then
die("--opt-binequiv is available only for PUC Lua 5.1!")
end
if #fspec > 0 then
if #fspec > 1 and option.OUTPUT_FILE then
die("with -o, only one source file can be specified")
end
do_files(fspec)
return true
else
die("nothing to do!")
end
end
-- entry point -> main() -> do_files()
if not main() then
die("Please run with option -h or --help for usage information")
end

View File

@@ -0,0 +1,300 @@
= Features and Usage
Kein-Hong Man
2011-09-13
== Features
LuaSrcDiet features include the following:
* Predefined default, _--basic_ (token-only) and _--maximum_ settings.
* Avoid deleting a block comment with a certain message with _--keep_; this is for copyright or license texts.
* Special handling for `#!` (shbang) lines and in functions, `self` implicit parameters.
* Dumping of raw information using _--dump-lexer_ and _--dump-parser_.
See the `samples` directory.
* A HTML plugin: outputs files that highlights globals and locals, useful for eliminating globals. See the `samples` directory.
* An SLOC plugin: counts significant lines of Lua code, like SLOCCount.
* Source and binary equivalence testing with _--opt-srcequiv_ and _--opt-binequiv_.
List of optimizations:
* Line endings are always normalized to LF, except those embedded in comments or strings.
* _--opt-comments_: Removal of comments and comment blocks.
* _--opt-whitespace_: Removal of whitespace, excluding end-of-line characters.
* _--opt-emptylines_: Removal of empty lines.
* _--opt-eols_: Removal of unnecessary end-of-line characters.
* _--opt-strings_: Rewrite strings and long strings. See the `samples` directory.
* _--opt-numbers_: Rewrite numbers. See the `samples` directory.
* _--opt-locals_: Rename local variable names. Does not rename field or method names.
* _--opt-entropy_: Tries to improve symbol entropy when renaming locals by calculating actual letter frequencies.
* _--opt-experimental_: Apply experimental optimizations.
LuaSrcDiet tries to allow each option to be enabled or disabled separately, but they are not completely orthogonal.
If comment removal is disabled, LuaSrcDiet only removes trailing whitespace.
Trailing whitespace is not removed in long strings, a warning is generated instead.
If empty line removal is disabled, LuaSrcDiet keeps all significant code on the same lines.
Thus, a user is able to debug using the original sources as a reference since the line numbering is unchanged.
String optimization deals mainly with optimizing escape sequences, but delimiters can be switched between single quotes and double quotes if the source size of the string can be reduced.
For long strings and long comments, LuaSrcDiet also tries to reduce the `=` separators in the
delimiters if possible.
For number optimization, LuaSrcDiet saves space by trying to generate the shortest possible sequence, and in the process it does not produce “proper” scientific notation (e.g. 1.23e5) but does away with the decimal point (e.g. 123e3) instead.
The local variable name optimizer uses a full parser of Lua 5.1 source code, thus it can rename all local variables, including upvalues and function parameters.
It should handle the implicit `self` parameter gracefully.
In addition, local variable names are either renamed into the shortest possible names following English frequent letter usage or are arranged by calculating entropy with the _--opt-entropy_ option.
Variable names are reused whenever possible, reducing the number of unique variable names.
For example, for `LuaSrcDiet.lua` (version 0.11.0), 683 local identifiers representing 88 unique names were optimized into 32 unique names, all which are one character in length, saving over 2600 bytes.
If you need some kind of reassurance that your app will still work at reduced size, see the section on verification below.
== Usage
LuaSrcDiet needs a Lua 5.1.x (preferably Lua 5.1.4) binary to run.
On Unix machines, one can use the following command line:
[source, sh]
LuaSrcDiet myscript.lua -o myscript_.lua
On Windows machines, the above command line can be used on Cygwin, or you can run Lua with the LuaSrcDiet script like this:
[source, sh]
lua LuaSrcDiet.lua myscript.lua -o myscript_.lua
When run without arguments, LuaSrcDiet prints a list of options.
Also, you can check the `Makefile` for some examples of command lines to use.
For example, for maximum code size reduction and maximum verbosity, use:
[source, sh]
LuaSrcDiet --maximum --details myscript.lua -o myscript_.lua
=== Output Example
A sample output of LuaSrcDiet 0.11.0 for processing `llex.lua` at _--maximum_ settings is as follows:
----
Statistics for: LuaSrcDiet.lua -> sample/LuaSrcDiet.lua
*** local variable optimization summary ***
----------------------------------------------------------
Variable Unique Decl. Token Size Average
Types Names Count Count Bytes Bytes
----------------------------------------------------------
Global 10 0 19 95 5.00
----------------------------------------------------------
Local (in) 88 153 683 3340 4.89
TOTAL (in) 98 153 702 3435 4.89
----------------------------------------------------------
Local (out) 32 153 683 683 1.00
TOTAL (out) 42 153 702 778 1.11
----------------------------------------------------------
*** lexer-based optimizations summary ***
--------------------------------------------------------------------
Lexical Input Input Input Output Output Output
Elements Count Bytes Average Count Bytes Average
--------------------------------------------------------------------
TK_KEYWORD 374 1531 4.09 374 1531 4.09
TK_NAME 795 3963 4.98 795 1306 1.64
TK_NUMBER 54 59 1.09 54 59 1.09
TK_STRING 152 1725 11.35 152 1717 11.30
TK_LSTRING 7 1976 282.29 7 1976 282.29
TK_OP 997 1092 1.10 997 1092 1.10
TK_EOS 1 0 0.00 1 0 0.00
--------------------------------------------------------------------
TK_COMMENT 140 6884 49.17 1 18 18.00
TK_LCOMMENT 7 1723 246.14 0 0 0.00
TK_EOL 543 543 1.00 197 197 1.00
TK_SPACE 1270 2465 1.94 263 263 1.00
--------------------------------------------------------------------
Total Elements 4340 21961 5.06 2841 8159 2.87
--------------------------------------------------------------------
Total Tokens 2380 10346 4.35 2380 7681 3.23
--------------------------------------------------------------------
----
Overall, the file size is reduced by more than 9 kiB.
Tokens in the above report can be classified into “real” or actual tokens, and “fake” or whitespace tokens.
The number of “real” tokens remained the same.
Short comments and long comments were completely eliminated.
The number of line endings was reduced by 59, while all but 152 whitespace characters were optimized away.
So, token separators (whitespace, including line endings) now takes up just 10 % of the total file size.
No optimization of number tokens was possible, while 2 bytes were saved for string tokens.
For local variable name optimization, the report shows that 38 unique local variable names were reduced to 20 unique names.
The number of identifier tokens should stay the same (there is currently no optimization option to optimize away non-essential or unused “real” tokens).
Since there can be at most 53 single-character identifiers, all local variables are now one character in length.
Over 600 bytes was saved.
_--details_ will give a longer report and much more information.
A sample output of LuaSrcDiet 0.12.0 for processing the one-file `LuaSrcDiet.lua` program itself at _--maximum_ and _--opt-experimental_ settings is as follows:
----
*** local variable optimization summary ***
----------------------------------------------------------
Variable Unique Decl. Token Size Average
Types Names Count Count Bytes Bytes
----------------------------------------------------------
Global 27 0 51 280 5.49
----------------------------------------------------------
Local (in) 482 1063 4889 21466 4.39
TOTAL (in) 509 1063 4940 21746 4.40
----------------------------------------------------------
Local (out) 55 1063 4889 4897 1.00
TOTAL (out) 82 1063 4940 5177 1.05
----------------------------------------------------------
*** BINEQUIV: binary chunks are sort of equivalent
Statistics for: LuaSrcDiet.lua -> app_experimental.lua
*** lexer-based optimizations summary ***
--------------------------------------------------------------------
Lexical Input Input Input Output Output Output
Elements Count Bytes Average Count Bytes Average
--------------------------------------------------------------------
TK_KEYWORD 3083 12247 3.97 3083 12247 3.97
TK_NAME 5401 24121 4.47 5401 7552 1.40
TK_NUMBER 467 494 1.06 467 494 1.06
TK_STRING 787 7983 10.14 787 7974 10.13
TK_LSTRING 14 3453 246.64 14 3453 246.64
TK_OP 6381 6861 1.08 6171 6651 1.08
TK_EOS 1 0 0.00 1 0 0.00
--------------------------------------------------------------------
TK_COMMENT 1611 72339 44.90 1 18 18.00
TK_LCOMMENT 18 4404 244.67 0 0 0.00
TK_EOL 4419 4419 1.00 1778 1778 1.00
TK_SPACE 10439 24475 2.34 2081 2081 1.00
--------------------------------------------------------------------
Total Elements 32621 160796 4.93 19784 42248 2.14
--------------------------------------------------------------------
Total Tokens 16134 55159 3.42 15924 38371 2.41
--------------------------------------------------------------------
* WARNING: before and after lexer streams are NOT equivalent!
----
The command line was:
[source, sh]
lua LuaSrcDiet.lua LuaSrcDiet.lua -o app_experimental.lua --maximum --opt-experimental --noopt-srcequiv
The important thing to note is that while the binary chunks are equivalent, the source lexer streams are not equivalent.
Hence, the _--noopt-srcequiv_ makes LuaSrcDiet report a warning for failing the source equivalence test.
`LuaSrcDiet.lua` was reduced from 157 kiB to about 41.3 kiB.
The _--opt-experimental_ option saves an extra 205 bytes over standard _--maximum_.
Note the reduction in `TK_OP` count due to a reduction in semicolons and parentheses.
`TK_SPACE` has actually increased a bit due to semicolons that are changed into single spaces; some of these spaces could not be removed.
For more performance numbers, see the <<performance-stats#, Performance Statistics>> page.
== Verification
Code size reduction can be quite a hairy thing (even I peer at the results in suspicion), so some kind of verification is desirable for users who expect processed files to _not_ blow up.
Since LuaSrcDiet has been talked about as a tool to reduce code size in projects such as WoW add-ons, `eLua` and `nspire`, adding a verification step will reduce risk for all users of LuaSrcDiet.
LuaSrcDiet performs two kinds of equivalence testing as of version 0.12.0.
The two tests can be very, very loosely termed as _source equivalence testing_ and _binary equivalence testing_.
They are controlled by the _--opt-srcequiv_ and _--opt-binequiv_ options and are enabled by default.
Testing behaviour can be summarized as follows:
* Both tests are always executed.
The options control the resulting actions taken.
* Both options are normally enabled.
This will make any failing test to throw an error.
* When an option is disabled, LuaSrcDiet will at most print a warning.
* For passing results, see the following subsections that describe what the tests actually does.
You only need to disable a testing option for experimental optimizations (see the following section for more information on this).
For anything up to and including _--maximum_, both tests should pass.
If any test fail under these conditions, then something has gone wrong with LuaSrcDiet, and I would be interested to know what has blown up.
=== _--opt-srcequiv_ Source Equivalence
The source equivalence test uses LuaSrcDiets lexer to read and compare the _before_ and _after_ lexer token streams.
Numbers and strings are dumped as binary chunks using `loadstring()` and `string.dump()` and the results compared.
If your file passes this test, it means that a Lua 5.1.x binary should see the exact same token streams for both _before_ and _after_ files.
That is, the parser in Lua will see the same lexer sequence coming from the source for both files and thus they _should_ be equivalent.
Touch wood.
Heh.
However, if you are _cross-compiling_, it may be possible for this test to fail.
Experienced Lua developers can modify `equiv.lua` to handle such cases.
=== _--opt-binequiv_ Binary Equivalence
The binary equivalence test uses `loadstring()` and `string.dump()` to generate binary chunks of the entire _before_ and _after_ files.
Also, any shbang (`#!`) lines are removed prior to generation of the binary chunks.
The binary chunks are then run through a fake `undump` routine to verify the integrity of the binary chunks and to compare all parts that ought to be identical.
On a per-function prototype basis (where _ignored_ means that any difference between the two binary chunks is ignored):
* All debug information is ignored.
* The source name is ignored.
* Any line number data is ignored.
For example, `linedefined` and `lastlinedefined`.
The rest of the two binary chunks must be identical.
So, while the two are not binary-exact, they can be loosely termed as “equivalent” and should run in exactly the same manner.
Sort of.
You get the idea.
This test may also cause problems if you are _cross-compiling_.
== Experimental Stuff
The _--opt-experimental_ option applies experimental optimizations that generally, makes changes to “real” tokens.
Such changes may or may not lead to the result failing binary chunk equivalence testing.
They would likely fail source lexer stream equivalence testing, so the _--noopt-srcequiv_ option needs to be applied so that LuaSrcDiet just gives a warning instead of an error.
For sample files, see the `samples` directory.
Currently implemented experimental optimizations are as follows:
=== Semicolon Operator Removal
The semicolon (`;`) operator is an optional operator that is used to separate statements.
The optimization turns all of these operators into single spaces, which are then run through whitespace removal.
At worst, there will be no change to file size.
* _Fails_ source lexer stream equivalence.
* _Passes_ binary chunk equivalence.
=== Function Call Syntax Sugar Optimization
This optimization turns function calls that takes a single string or long string parameter into its syntax-sugar representation, which leaves out the parentheses.
Since strings can abut anything, each instance saves 2 bytes.
For example, the following:
[source, lua]
fish("cow")fish('cow')fish([[cow]])
is turned into:
[source, lua]
fish"cow"fish'cow'fish[[cow]]
* _Fails_ source lexer stream equivalence.
* _Passes_ binary chunk equivalence.
=== Other Experimental Optimizations
There are two more of these optimizations planned, before focus is turned to the Lua 5.2.x series:
* Simple `local` keyword removal.
Planned to work for a few kinds of patterns only.
* User directed name replacement, which will need user input to modify names or identifiers used in table keys and function methods or fields.

View File

@@ -0,0 +1,128 @@
= Performance Statistics
Kein-Hong Man
2011-09-13
== Size Comparisons
The following is the result of processing `llex.lua` from LuaSrcDiet 0.11.0 using various optimization options:
|===
| LuaSrcDiet Option | Size (bytes)
| Original | 12,421
| Empty lines only | 12,395
| Whitespace only | 9,372
| Local rename only | 11,794
| _--basic_ setting | 3,835
| Program default | 3,208
| _--maximum_ setting | 3,130
|===
The programs default settings does not remove all unnecessary EOLs.
The _--basic_ setting is more conservative than the default settings, it disables optimization of strings and numbers and renaming of locals.
For version 0.12.0, the following is the result of processing `LuaSrcDiet.lua` using various optimization options:
|===
| LuaSrcDiet Option | Size (bytes)
| Original | 160,796
| _--basic_ setting | 60,219
| Program default | 43,650
| _--maximum_ setting | 42,453
| max + experimental | 42,248
|===
The above best size can go a lot lower with simple `local` keyword removal and user directed name replacement, which will be the subject of the next release of LuaSrcDiet.
== Compression and luac
File sizes of LuaSrcDiet 0.11.0 main files in various forms:
[cols="m,5*d", options="header,footer"]
|===
| Source File | Original Size (bytes) | `luac` normal (bytes) | `luac` stripped (bytes) | LuaSrcDiet _--basic_ (bytes) | LuaSrcDiet _--maximum_ (bytes)
| LuaSrcDiet.lua | 21,961 | 20,952 | 11,000 | 11,005 | 8,159
| llex.lua | 12,421 | 8,613 | 4,247 | 3,835 | 3,130
| lparser.lua | 41,757 | 27,215 | 12,506 | 11,755 | 7,666
| optlex.lua | 31,009 | 16,992 | 8,021 | 9,129 | 6,858
| optparser.lua | 16,511 | 9,021 | 3,520 | 5,087 | 2,999
| Total | 123,659 | 82,793 | 39,294 | 40,811 | 28,812
|===
* “LuaSrcDiet --maximum” has the smallest total file size.
* The ratio of “Original Size” to “LuaSrcDiet --maximum” is *4.3*.
* The ratio of “Original Size” to “luac stripped” is *3.1*.
* The ratio of “luac stripped” to “LuaSrcDiet --maximum” is *1.4*.
Compressibility of LuaSrcDiet 0.11.0 main files in various forms:
|===
| Compression Method | Original Size | `luac` normal | `luac` stripped | LuaSrcDiet _--basic_ | LuaSrcDiet _--maximum_
| Uncompressed originals | 123,659 | 82,793 | 39,294 | 40,811 | 28,812
| gzip -9 | 28,288 | 29,210 | 17,732 | 12,041 | 10,451
| bzip2 -9 | 24,407 | 27,232 | 16,856 | 11,480 | 9,815
| lzma (7-zip max) | 25,530 | 23,908 | 15,741 | 11,241 | 9,685
|===
* “LuaSrcDiet --maximum” has the smallest total file size (but a binary chunk loads faster and works with a smaller Lua executable).
* The ratio of “Original size” to “Original size + bzip2” is *5.1*.
* The ratio of “Original size” to “LuaSrcDiet --maximum + bzip2” is *12.6*.
* The ratio of “LuaSrcDiet --maximum” to “LuaSrcDiet --maximum + bzip2” is *2.9*.
* The ratio of “Original size” to “luac stripped + bzip2” is *7.3*.
* The ratio of “luac stripped” to “luac stripped + bzip2” is *2.3*.
* The ratio of “luac stripped + bzip2” to “LuaSrcDiet --maximum + bzip2” is *1.7*.
So, squeezed source code are smaller than stripped binary chunks and compresses better than stripped binary chunks, at a ratio of 2.9 for squeezed source code versus 2.3 for stripped binary chunks.
Compressed binary chunks is still a very efficient way of storing Lua scripts, because using only binary chunks allow for the parts of Lua needed to compile from sources to be omitted (`llex.o`, `lparser.o`, `lcode.o`, `ldump.o`), saving over 24KB in the process.
Note that LuaSrcDiet _does not_ answer the question of whether embedding source code is better or embedding binary chunks is better.
It is simply a utility for producing smaller source code files and an exercise in processing Lua source code using a Lua-based lexer and parser skeleton.
== Compile Speed
The following is a primitive attempt to analyze in-memory Lua script loading performance (using the `loadstring` function in Lua).
The LuaSrcDiet 0.11.0 files (original, squeezed with _--maximum_ and stripped binary chunks versions) are loaded into memory first before a loop runs to repeatedly load the script files for 10 seconds.
A null loop is also performed (processing empty strings) and the time taken per null iteration is subtracted as a form of null adjustment.
Then, various performance parameters are calculated.
Note that `LuaSrcDiet.lua` was slightly modified (`#!` line removed) to let the `loadstring` function run.
The results below were obtained with a Lua 5.1.3 executable compiled using `make generic` on Cygwin/Windows XP SP2 on a Sempron 3000+ (1.8GHz).
The LuaSrcDiet 0.11.0 source files have 11,180 “real” tokens in total.
[cols="<h,4*d", options="header"]
|===
| | Null loop | Stripped binary chunk | Original Sources | Squeezed Sources
| Total Size (bytes) | 0 | 39,294 | 123,640 | 28,793
| Iterations | 312,155 | 9,680 | 1306 | 1,592
| Duration (sec) | 10 | 10 | 10 | 10
| Time/iteration (msec) | 0.032 | 1.033 | 7.657 | 6.281
| _Time/iteration, null adjusted (msec)_ | | 1.001 | 7.625 | 6.249
| _Load rate (MiB/sec)_ | | 37.44 | 15.46 | 4.39
| Load time per byte (ns) | | 25.5 | 61.7 | 217.0
| Load time per token (ns) | | | 682 | 559
| Source time vs binary chunk time ratio | | 1.00 | 7.62 | 6.24
| Binary chunk rate vs. source rate ratio | | 1.00 | 2.42 | 8.53
|===
The above shows that stripped binary chunks is still, in many ways, the highest-performance form of fixed Lua scripts.
On a very average machine, scripts load at over 37 MiB/sec (in memory).
This is very comparable to the burst speeds of common desktop hard disks of 2008.
If instant response is paramount, stripped binary chunks has little competition.
By contrast, source code that is squeezed to the maximum using LuaSrcDiet can only muster an in-memory load rate of 4.4 MiB/sec.
The original sources load at about 15.5 MiB/sec, but most of the speed is from the lexer scanning over comments and whitespace.
A quick calculation indicates that the speed of the lexer over comments and whitespace can be as much as 65 MiB/sec, but note that the speed is all for naught.
What really matters are the real tokens, and the squeezed source code manages to load faster than the original sources by 18 %.
So, the loading of stripped binary chunks is faster than squeezed source code by a bit over 6×.
The 4.4 MiB/sec speed for squeezed source code is still quite respectable.
When an application considers the time taken to load data from the disk and perhaps the time taken to decompress, loading source code may be perfectly fine in terms of performance.
For programs that already embed source code, using LuaSrcDiet to squeeze the source code probably speeds loading up by a tiny bit in addition to making programs smaller.

View File

@@ -0,0 +1,386 @@
= Technical Notes
Kein-Hong Man
2011-09-13
== Lexer Notes
The lexer (`llex.lua`) is a version of the native 5.1.x lexer from Yueliang 0.4.0, with significant modifications.
It does have several limitations:
* The decimal point must be `.` (period).
There is no localized decimal point replacement magic.
* There is no support for nested `[[`...`]]` long strings (no `LUA_COMPAT_LSTR`).
* The lexer may not properly lex source code with characters beyond the normal ASCII character set.
Identifiers with accented characters (or any character beyond a byte value of 127) cannot be recognized.
Instead of returning one token on each call, `llex.lua` processes an entire string (all data from an entire file) and returns.
Two lists (tokens and semantic information items) are set up in the module for use by the caller.
For maximum flexibility during processing, the lexer returns non-grammar lexical elements as tokens too.
Non-grammar elements, such as comments, whitespace, line endings, are classified along with “normal” tokens.
The lexer classifies 7 kinds of grammar tokens and 4 kinds of non-grammar tokens, as follows:
[cols="m,d"]
|===
| Grammar Token | Description
| TK_KEYWORD | keywords
| TK_NAME | identifiers
| TK_NUMBER | numbers (unconverted, kept in original form)
| TK_STRING | strings (no translation is done, includes delimiters)
| TK_LSTRING | long strings (no translation is done, includes delimiters)
| TK_OP | operators and punctuation (most single-char, some double)
| TK_EOS | end-of-stream (there is only one for each file/stream)
|===
[cols="m,d"]
|===
| Whitespace Token | Description
| TK_SPACE | whitespace (generally, spaces, \t, \v and \f)
| TK_COMMENT | comments (includes delimiters, also includes special first line shbang, which is handled specially in the optimizer)
| TK_LCOMMENT | block comments (includes delimiters)
| TK_EOL | end-of-lines (excludes those embedded in strings)
|===
A list of tokens can be generated by using the _--dump-lexer_ option, like this:
[source, sh]
lua LuaSrcDiet.lua --dump-lexer llex.lua > dump_llex.dat
== Lexer Optimizations
We aim to keep lexer-based optimizations free of parser considerations, i.e. we allow for generalized optimization of token sequences.
The table below considers the requirements for all combinations of significant tokens (except `TK_EOS`).
Other tokens are whitespace-like.
Comments can be considered to be a special kind of whitespace, e.g. a short comment needs to have a following EOL token, if we do not want to optimize away short comments.
[cols="h,6*m", options="header"]
|===
| _1st  2nd Token_ | Keyword | Name | Number | String | LString | Oper
| Keyword | [S] | [S] | [S] | | |
| Name | [S] | [S] | [S] | | |
| Number | [S] | [S] | [S] | | | [1]
| String | | | | | |
| LString | | | | | |
| Oper | | | [1] | | | [2]
|===
A dash (`-`) in the above means that the first token can abut the second token.
`*[S]*`:: Need at least one whitespace, set as either a space or kept as an EOL.
`*[1]*`::
Need a space if operator is a `.`, all others okay.
A `+` or `-` is used as part of a floating-point spec, but there does not appear to be any way of creating a float by joining with number with a `+` or `-` plus another number.
Since an `e` has to be somewhere in the first token, this cant be done.
`*[2]*`::
Normally there cannot be consecutive operators, but we plan to allow for generalized optimization of token sequences, i.e. even sequences that are grammatically illegal; so disallow adjacent operators if:
* the first is in `[=<>]` and the second is `=`
* disallow dot sequences to be adjacent, but `...` first okay
* disallow `[` followed by `=` or `[` (not optimal)
Also, a minus `-` cannot preceed a Comment or LComment, because comments start with a `--` prefix.
Apart from that, all Comment or LComment tokens can be set abut with a real token.
== Local Variable Renaming
The following discusses the problem of local variable optimization, specifically _local variable renaming_ in order to reduce source code size.
=== TK_NAME Token Considerations
A `TK_NAME` token means a number of things, and some of these cannot be renamed without analyzing the source code.
We are interested in the use of `TK_NAME` in the following:
[loweralpha]
. global variable access,
. local variable declaration, including `local` statements, `local` functions, function parameters, implicit `self` locals,
. local variable access, including upvalue access.
`TK_NAME` is also used in parts of the grammar as constant strings these tokens cannot be optimized without user assistance.
These include usage as:
[loweralpha, start=4]
. keys in `key=value` pairs in table construction,
. field or method names in `a:b` or `a.b` syntax forms.
For the local variable name optimization scheme used, we do not consider (d) and (e), and while global variables cannot be renamed without some kind of user assistance, they need to be considered or tracked as part of Luas variable access scheme.
=== Lifetime of a Local Variable
Consider the following example:
[source, lua]
local string, table = string, table
In the example, the two locals are assigned the values of the globals with the same names.
When Lua encounters the declaration portion:
[source, lua]
local string, table
the parser cannot immediately make the two local variable available to following code.
In the parser and code generator, locals are inactive when entries are created.
They are activated only when the function `adjustlocalvars()` is called to activate the appropriate local variables.
NOTE: The terminology used here may not be identical to the ones used in the Dragon Book they merely follow the LuaSrcDiet code as it was written before I have read the Dragon Book.
In the example, the two local variables are activated only after the whole statement has been parsed, that is, after the last `table` token.
Hence, the statement works as expected.
Also, once the two local variables goes out of scope, `removevars()` is called to deactivate them, allowing other variables of the same name to become visible again.
Another example worth mentioning is:
[source, lua]
local a, a, a, = 1, 2, 3
The above will assign 3 to `a`.
Thus, when optimizing local variable names, (1) we need to consider accesses of global variable names affecting the namespace, (2) for the local variable names themselves, we need to consider when they are declared, activated and removed, and (3) within the “live” time of locals, we need to know when they are accessed (since locals that are never accessed dont really matter.)
=== Local Variable Tracking
Every local variable declaration is considered an object to be renamed.
From the parser, we have the original name of the local variable, the token positions for declaration, activation and removal, and the token position for all the `TK_NAME` tokens which references this local.
All instances of the implicit `self` local variable are also flagged as such.
In addition to local variable information, all global variable accesses are tabled, one object entry for one name, and each object has a corresponding list of token positions for the `TK_NAME` tokens, which is where the global variables were accessed.
The key criteria is: *Our act of renaming cannot change the visibility of any of these locals and globals at the time they are accessed*.
However, _their scope of visibility may be changed during which they are not accessed_, so someone who tries to insert a variable reference somewhere into a program that has its locals renamed may find that it now refers to a different variable.
Of course, if every variable has a unique name, then there is no need for a name allocation algorithm, as there will be no conflict.
But, in order to maximize utilization of short identifier names to reduce the final code size, we want to reuse the names as much as possible.
In addition, fewer names will likely reduce symbol entropy and may slightly improve compressibility of the source code.
LuaSrcDiet avoids the use of non-ASCII letters, so there are only 53 single-character variable names.
=== Name Allocation Theory
To understand the renaming algorithm, first we need to establish how different local and global variables can operate happily without interfering with each other.
Consider three objects, local object A, local object B and global object G.
A and B involve declaration, activation and removal, and within the period it is active, there may be zero or more accesses of the local.
For G, there are only global variable accesses to look into.
Assume that we have assigned a new name to A and we wish to consider its effects on other locals and globals, for which we choose B and G as examples.
We assume local B has not been assigned a new name as we expect our algorithm to take care of collisions.
As lifetime is something like this:
----
Decl Act Rem
+ +-------------------------------+
-------------------------------------------------
----
where “Decl” is the time of declaration, “Act” is the time of activation, and “Rem” is the time of removal.
Between “Act” and “Rem”, the local is alive or “live” and Lua can see it if its corresponding `TK_NAME` identifier comes up.
----
Decl Act Rem
+ +-------------------------------+
-------------------------------------------------
* * * *
(1) (2) (3) (4)
----
Recall that the key criteria is to not change the visibility of globals and locals during when they are accessed.
Consider local and global accesses at (1), (2), (3) and (4).
A global G of the same name as A will only collide at (3), where Lua will see A and not G.
Since G must be accessed at (3) according to what the parser says, and we cannot modify the positions of “Decl”, “Act” and “Rem”, it follows that A cannot have the same name as G.
----
Decl Act Rem
+ +-----------------------+
---------------------------------
(1)+ +---+ (2)+ +---+ (3)+ +---+ (4)+ +---+
--------- --------- --------- ---------
----
For the case of A and B having the same names and colliding, consider the cases for which B is at (1), (2), (3) or (4) in the above.
(1) and (4) means that A and B are completely isolated from each other, hence in the two cases, A and B can safely use the same variable names.
To be specific, since we have assigned A, B is considered completely isolated from A if Bs activation-to-removal period is isolated from the time of As first access to last access, meaning Bs active time will never affect any of As accesses.
For (2) and (3), we have two cases where we need to consider which one has been activated first.
For (2), B is active before A, so A cannot impose on B.
But As accesses are valid while B is active, since A can override B.
For no collision in the case of (2), we simply need to ensure that the last access of B occurs before A is activated.
For (3), B is activated before A, hence B can override As accesses.
For no collision, all of As accesses cannot happen while B is active.
Thus position (3) follows the “A is never accessed when B is active” rule in a general way.
Local variables of a child function are in the position of (3).
To illustrate, the local B can use the same name as local A and live in a child function or block scope if each time A is accessed, Lua sees A and not B.
So we have to check all accesses of A and see whether they collide with the active period of B.
If A is not accessed during that period, then B can be active with the same name.
The above appears to resolve all sorts of cases where the active times of A and B overlap.
Note that in the above, the allocator does not need to know how locals are separated according to function prototypes.
Perhaps the allocator can be simplified if knowledge of function structure is utilized.
This scheme was implemented in a hurry in 2008 — it could probably be simpler if Lua grammar is considered, but LuaSrcDiet mainly processes various index values in tables.
=== Name Allocation Algorithm
To begin with, the name generator is mostly separate from the name allocation algorithm.
The name generator returns the next shortest name for the algorithm to apply to local variables.
To attempt to reduce symbol entropy (which benefit compression algorithms), the name generator follows English frequent letter usage.
There is also an option to calculate an actual symbol entropy table from the input data.
Since there are 53 one-character identifiers and (53 * 63 - 4) two-character identifiers (minus a few keywords), there isnt a pressing need to optimally maximize name reuse.
The single-file version of LuaSrcDiet 0.12.0, at just over 3000 SLOC and 156 kiB in size, currently allocates around 55 unique local variable names.
In theory, we should need no more than 260 local identifiers by default.
Why?
Since `LUAI_MAXVARS` is 200 and `LUAI_MAXUPVALUES` is 60, at any block scope, there can be at most `(LUAI_MAXVARS + LUAI_MAXUPVALUES)` locals referenced, or 260.
Also, those from outer scopes not referenced in inner scopes can reuse identifiers.
The net effect of this is that a local variable name allocation method should not allocate more than 260 identifier names for locals.
The current algorithm is a simple first-come first-served scheme:
[loweralpha]
. One local object that use the most tokens is named first.
. Any other non-conflicting locals with respect to the first object are assigned the same name.
. Assigned locals are removed from consideration and the procedure is repeated for objects that have not been assigned new names.
. Steps (a) to (c) repeats until no local objects are left.
In addition, there are a few extra issues to take care of:
[loweralpha, start=5]
. Implicit `self` locals that have been flagged as such are already “assigned to” and so they are left unmodified.
. The name generator skips `self` to avoid conflicts.
This is not optimal but it is unlikely a script will use so many local variables as to reach `self`.
. Keywords are also skipped for the name generator.
. Global name conflict resolution.
For (h), global name conflict resolution is handled just after the new name is generated.
The name can still be used for some locals even if it conflicts with other locals.
To remove conflicts, global variable accesses for the particular identifier name is checked.
Any local variables that are active when a global access is made is marked to be skipped.
The rest of the local objects can then use that name.
The algorithm has additional code for handling locals that use the same name in the same scope.
This extends the basic algorithm that was discussed earlier.
For example:
[source, lua]
----
local foo = 10 -- <1>
...
local foo = 20 -- <2>
...
print(e)
----
Since we are considering name visibility, the first `foo` does not really cease to exist when the second `foo` is declared, because if we were to make that assumption, and the first `foo` is removed before (2), then I should be able to use `e` as the name for the first `foo` and after (2), it should not conflict with variables in the outer scope with the same name.
To illustrate:
[source, lua]
----
local e = 10 -- 'foo' renamed to 'e'
...
local t = 20 -- error if we assumed 'e' removed here
...
print(e)
----
Since `e` is a global in the example, we now have an error as the name as been taken over by a local.
Thus, the first `foo` local must have its active time extend to the end of the current scope.
If there is no conflict between the first and second `foo`, the algorithm may still assign the same names to them.
The current fix to deal with the above chains local objects in order to find the removal position.
It may be possible to handle this in a clean manner LuaSrcDiet handles it as a fix to the basic algorithm.
== Ideas
The following is a list of optimization ideas that do not require heavy-duty source code parsing and comprehension.
=== Lexer-Based Optimization Ideas
* Convert long strings to normal strings, vice versa. +
_A little desperate for a few bytes, can be done, but not real keen on implementing it._
* Special number forms to take advantage of constant number folding. +
_For example, 65536 can be represented using 2^16^, and so on.
An expression must be evaluated in the same way, otherwise this seems unsafe._
* Warn if a number has too many digits. +
_Should we warn or “test and truncate”?
Not really an optimization that will see much use._
* Warn of opportunity for using a `local` to zap a bunch of globals. +
_Current recommendation is to use the HTML plugin to display globals in red.
The developer can then visually analyze the source code and make the appropriate fixes.
I think this is better than having the program guess the intentions of the developer._
* Spaces to tabs in comments, long comments, or long strings. +
_For long strings, need to know users intention.
Would rather not implement._
=== Parser-Based Optimization Ideas
Heavy-duty optimizations will need more data to be generated by the parser.
A full AST may eventually be needed.
The most attractive idea that can be quickly implemented with a significant code size “win” is to reduce the number of `local` keywords.
* Remove unused ``local``s that can be removed in the source. +
_Need to consider unused ``local``s in multiple assignments._
* Simplify declaration of ``local``s that can be merged. +
_From:_
+
[source, lua]
----
-- separate locals
local foo
local bar
-- separate locals with assignments
local foo = 123
local bar = "pqr"
----
+
_To:_
+
[source, lua]
----
-- merged locals
local foo,bar
-- merged locals with assignments
local foo,bar=123,"pqr"
----
* Simplify declarations using `nil`. +
_From:_
[source, lua]
local foo, bar = nil, nil
+
_To:_
[source, lua]
local foo,bar
* Simplify ``return``s using `nil`. +
_How desirable is this? From Lua list discussions, it seems to be potentially unsafe unless all return locations are known and checked._
* Removal of optional semicolons in statements and removal of commas or semicolons in table constructors. +
_Yeah, this might save a few bytes._
* Remove table constructor elements using `nil`. +
_Not sure if this is safe to do._
* Simplify logical or relational operator expressions. +
_This is more suitable for an optimizing compiler project._

View File

@@ -0,0 +1,41 @@
-- vim: set ft=lua:
package = 'LuaSrcDiet'
version = '0.3.0-2'
source = { url = 'https://github.com/jirutka/luasrcdiet/archive/v0.3.0/luasrcdiet-0.3.0.tar.gz', md5 = 'c0ff36ef66cd0568c96bc54e9253a8fa' }
description = {
summary = 'Compresses Lua source code by removing unnecessary characters',
detailed = [[
This is revival of LuaSrcDiet originally written by Kein-Hong Man.]],
homepage = 'https://github.com/jirutka/luasrcdiet',
maintainer = 'Jakub Jirutka <jakub@jirutka.cz>',
license = 'MIT',
}
dependencies = {
'lua >= 5.1',
}
build = {
type = 'builtin',
modules = {
['luasrcdiet'] = 'luasrcdiet/init.lua',
['luasrcdiet.equiv'] = 'luasrcdiet/equiv.lua',
['luasrcdiet.fs'] = 'luasrcdiet/fs.lua',
['luasrcdiet.llex'] = 'luasrcdiet/llex.lua',
['luasrcdiet.lparser'] = 'luasrcdiet/lparser.lua',
['luasrcdiet.optlex'] = 'luasrcdiet/optlex.lua',
['luasrcdiet.optparser'] = 'luasrcdiet/optparser.lua',
['luasrcdiet.plugin.example'] = 'luasrcdiet/plugin/example.lua',
['luasrcdiet.plugin.html'] = 'luasrcdiet/plugin/html.lua',
['luasrcdiet.plugin.sloc'] = 'luasrcdiet/plugin/sloc.lua',
['luasrcdiet.utils'] = 'luasrcdiet/utils.lua',
},
install = {
bin = {
luasrcdiet = 'bin/luasrcdiet',
}
}
}

View File

@@ -0,0 +1,28 @@
rock_manifest = {
bin = {
luasrcdiet = "6c318685d57f827cf5baf7037a5d6072"
},
doc = {
["features-and-usage.adoc"] = "157587c27a0c340d9d1dd06af9b339b5",
["performance-stats.adoc"] = "cf5f96a86e021a3a584089fafcabd056",
["tech-notes.adoc"] = "075bc34e667a0055e659e656baa2365a"
},
lua = {
luasrcdiet = {
["equiv.lua"] = "967a6b17573d229e326dbb740ad7fe8c",
["fs.lua"] = "53db7dfc50d026b683fad68ed70ead0f",
["init.lua"] = "c6f368e6cf311f3257067fed0fbcd06a",
["llex.lua"] = "ede897af261fc362a82d87fbad91ea2b",
["lparser.lua"] = "c1e1f04d412b79a040fd1c2b74112953",
["optlex.lua"] = "7c986da991a338494c36770b4a30fa9f",
["optparser.lua"] = "b125a271ac1c691dec68b63019b1b5da",
plugin = {
["example.lua"] = "86b5c1e9dc7959db6b221d6d5a0db3d1",
["html.lua"] = "c0d3336a133f0c8663f395ee98d54f6a",
["sloc.lua"] = "fb1a91b18b701ab83f21c87733be470a"
},
["utils.lua"] = "bd6c1e85c6a9bf3383d336a4797fb292"
}
},
["luasrcdiet-0.3.0-2.rockspec"] = "da70047e1b0cbdc1ff08d060327fa110"
}

View File

@@ -0,0 +1,650 @@
commands = {
luadocumentor = {
"luadocumentor/0.1.5-1"
},
luasrcdiet = {
"luasrcdiet/0.3.0-2"
}
}
dependencies = {
luadocumentor = {
["0.1.5-1"] = {
{
constraints = {
{
op = "~>",
version = {
5, 1, string = "5.1"
}
}
},
name = "lua"
},
{
constraints = {
{
op = "~>",
version = {
1, 6, string = "1.6"
}
}
},
name = "luafilesystem"
},
{
constraints = {
{
op = "~>",
version = {
0, 32, string = "0.32"
}
}
},
name = "markdown"
},
{
constraints = {
{
op = "~>",
version = {
0, 7, string = "0.7"
}
}
},
name = "metalua-compiler"
},
{
constraints = {
{
op = "~>",
version = {
0, 9, string = "0.9"
}
}
},
name = "penlight"
}
}
},
luafilesystem = {
["1.6.3-2"] = {
{
constraints = {
{
op = ">=",
version = {
5, 1, string = "5.1"
}
}
},
name = "lua"
}
}
},
luasrcdiet = {
["0.3.0-2"] = {
{
constraints = {
{
op = ">=",
version = {
5, 1, string = "5.1"
}
}
},
name = "lua"
}
}
},
markdown = {
["0.32-2"] = {
{
constraints = {
{
op = ">=",
version = {
5, 1, string = "5.1"
}
}
},
name = "lua"
}
}
},
["metalua-compiler"] = {
["0.7.3-1"] = {
{
constraints = {
{
op = "~>",
version = {
5, 1, string = "5.1"
}
}
},
name = "lua"
},
{
constraints = {
{
op = "~>",
version = {
1, 6, string = "1.6"
}
}
},
name = "luafilesystem"
},
{
constraints = {
{
op = ">=",
version = {
0, 7, 3, string = "0.7.3"
}
}
},
name = "metalua-parser"
}
}
},
["metalua-parser"] = {
["0.7.3-2"] = {
{
constraints = {
{
op = ">=",
version = {
5, 1, string = "5.1"
}
}
},
name = "lua"
}
}
},
penlight = {
["0.9.8-1"] = {
{
constraints = {},
name = "luafilesystem"
}
}
}
}
modules = {
defaultcss = {
"luadocumentor/0.1.5-1"
},
docgenerator = {
"luadocumentor/0.1.5-1"
},
extractors = {
"luadocumentor/0.1.5-1"
},
["fs.lfs"] = {
"luadocumentor/0.1.5-1"
},
lddextractor = {
"luadocumentor/0.1.5-1"
},
lfs = {
"luafilesystem/1.6.3-2"
},
luasrcdiet = {
"luasrcdiet/0.3.0-2"
},
["luasrcdiet.equiv"] = {
"luasrcdiet/0.3.0-2"
},
["luasrcdiet.fs"] = {
"luasrcdiet/0.3.0-2"
},
["luasrcdiet.llex"] = {
"luasrcdiet/0.3.0-2"
},
["luasrcdiet.lparser"] = {
"luasrcdiet/0.3.0-2"
},
["luasrcdiet.optlex"] = {
"luasrcdiet/0.3.0-2"
},
["luasrcdiet.optparser"] = {
"luasrcdiet/0.3.0-2"
},
["luasrcdiet.plugin.example"] = {
"luasrcdiet/0.3.0-2"
},
["luasrcdiet.plugin.html"] = {
"luasrcdiet/0.3.0-2"
},
["luasrcdiet.plugin.sloc"] = {
"luasrcdiet/0.3.0-2"
},
["luasrcdiet.utils"] = {
"luasrcdiet/0.3.0-2"
},
markdown = {
"markdown/0.32-2"
},
["metalua.compiler"] = {
"metalua-parser/0.7.3-2"
},
["metalua.compiler.bytecode"] = {
"metalua-compiler/0.7.3-1"
},
["metalua.compiler.bytecode.compile"] = {
"metalua-compiler/0.7.3-1"
},
["metalua.compiler.bytecode.lcode"] = {
"metalua-compiler/0.7.3-1"
},
["metalua.compiler.bytecode.ldump"] = {
"metalua-compiler/0.7.3-1"
},
["metalua.compiler.bytecode.lopcodes"] = {
"metalua-compiler/0.7.3-1"
},
["metalua.compiler.globals"] = {
"metalua-compiler/0.7.3-1"
},
["metalua.compiler.parser"] = {
"metalua-parser/0.7.3-2"
},
["metalua.compiler.parser.annot.generator"] = {
"metalua-parser/0.7.3-2"
},
["metalua.compiler.parser.annot.grammar"] = {
"metalua-parser/0.7.3-2"
},
["metalua.compiler.parser.expr"] = {
"metalua-parser/0.7.3-2"
},
["metalua.compiler.parser.ext"] = {
"metalua-parser/0.7.3-2"
},
["metalua.compiler.parser.lexer"] = {
"metalua-parser/0.7.3-2"
},
["metalua.compiler.parser.meta"] = {
"metalua-parser/0.7.3-2"
},
["metalua.compiler.parser.misc"] = {
"metalua-parser/0.7.3-2"
},
["metalua.compiler.parser.stat"] = {
"metalua-parser/0.7.3-2"
},
["metalua.compiler.parser.table"] = {
"metalua-parser/0.7.3-2"
},
["metalua.grammar.generator"] = {
"metalua-parser/0.7.3-2"
},
["metalua.grammar.lexer"] = {
"metalua-parser/0.7.3-2"
},
["metalua.loader"] = {
"metalua-compiler/0.7.3-1"
},
["metalua.pprint"] = {
"metalua-parser/0.7.3-2"
},
["metalua/compiler/ast_to_src.mlua"] = {
"metalua-compiler/0.7.3-1"
},
["metalua/extension/comprehension.mlua"] = {
"metalua-compiler/0.7.3-1"
},
["metalua/extension/match.mlua"] = {
"metalua-compiler/0.7.3-1"
},
["metalua/repl.mlua"] = {
"metalua-compiler/0.7.3-1"
},
["metalua/treequery.mlua"] = {
"metalua-compiler/0.7.3-1"
},
["metalua/treequery/walk.mlua"] = {
"metalua-compiler/0.7.3-1"
},
["models.apimodel"] = {
"luadocumentor/0.1.5-1"
},
["models.apimodelbuilder"] = {
"luadocumentor/0.1.5-1"
},
["models.internalmodel"] = {
"luadocumentor/0.1.5-1"
},
["models.ldparser"] = {
"luadocumentor/0.1.5-1"
},
["models/internalmodelbuilder.mlua"] = {
"luadocumentor/0.1.5-1"
},
pl = {
"penlight/0.9.8-1"
},
["pl.Date"] = {
"penlight/0.9.8-1"
},
["pl.List"] = {
"penlight/0.9.8-1"
},
["pl.Map"] = {
"penlight/0.9.8-1"
},
["pl.MultiMap"] = {
"penlight/0.9.8-1"
},
["pl.OrderedMap"] = {
"penlight/0.9.8-1"
},
["pl.Set"] = {
"penlight/0.9.8-1"
},
["pl.app"] = {
"penlight/0.9.8-1"
},
["pl.array2d"] = {
"penlight/0.9.8-1"
},
["pl.class"] = {
"penlight/0.9.8-1"
},
["pl.comprehension"] = {
"penlight/0.9.8-1"
},
["pl.config"] = {
"penlight/0.9.8-1"
},
["pl.data"] = {
"penlight/0.9.8-1"
},
["pl.dir"] = {
"penlight/0.9.8-1"
},
["pl.file"] = {
"penlight/0.9.8-1"
},
["pl.func"] = {
"penlight/0.9.8-1"
},
["pl.input"] = {
"penlight/0.9.8-1"
},
["pl.lapp"] = {
"penlight/0.9.8-1"
},
["pl.lexer"] = {
"penlight/0.9.8-1"
},
["pl.luabalanced"] = {
"penlight/0.9.8-1"
},
["pl.operator"] = {
"penlight/0.9.8-1"
},
["pl.path"] = {
"penlight/0.9.8-1"
},
["pl.permute"] = {
"penlight/0.9.8-1"
},
["pl.platf.luajava"] = {
"penlight/0.9.8-1"
},
["pl.pretty"] = {
"penlight/0.9.8-1"
},
["pl.seq"] = {
"penlight/0.9.8-1"
},
["pl.sip"] = {
"penlight/0.9.8-1"
},
["pl.strict"] = {
"penlight/0.9.8-1"
},
["pl.stringio"] = {
"penlight/0.9.8-1"
},
["pl.stringx"] = {
"penlight/0.9.8-1"
},
["pl.tablex"] = {
"penlight/0.9.8-1"
},
["pl.template"] = {
"penlight/0.9.8-1"
},
["pl.test"] = {
"penlight/0.9.8-1"
},
["pl.text"] = {
"penlight/0.9.8-1"
},
["pl.utils"] = {
"penlight/0.9.8-1"
},
["pl.xml"] = {
"penlight/0.9.8-1"
},
["template.file"] = {
"luadocumentor/0.1.5-1"
},
["template.index"] = {
"luadocumentor/0.1.5-1"
},
["template.index.recordtypedef"] = {
"luadocumentor/0.1.5-1"
},
["template.item"] = {
"luadocumentor/0.1.5-1"
},
["template.page"] = {
"luadocumentor/0.1.5-1"
},
["template.recordtypedef"] = {
"luadocumentor/0.1.5-1"
},
["template.usage"] = {
"luadocumentor/0.1.5-1"
},
["template.utils"] = {
"luadocumentor/0.1.5-1"
},
templateengine = {
"luadocumentor/0.1.5-1"
}
}
repository = {
luadocumentor = {
["0.1.5-1"] = {
{
arch = "installed",
commands = {
luadocumentor = "luadocumentor"
},
dependencies = {
luafilesystem = "1.6.3-2",
markdown = "0.32-2",
["metalua-compiler"] = "0.7.3-1",
["metalua-parser"] = "0.7.3-2",
penlight = "0.9.8-1"
},
modules = {
defaultcss = "defaultcss.lua",
docgenerator = "docgenerator.lua",
extractors = "extractors.lua",
["fs.lfs"] = "fs/lfs.lua",
lddextractor = "lddextractor.lua",
["models.apimodel"] = "models/apimodel.lua",
["models.apimodelbuilder"] = "models/apimodelbuilder.lua",
["models.internalmodel"] = "models/internalmodel.lua",
["models.ldparser"] = "models/ldparser.lua",
["models/internalmodelbuilder.mlua"] = "models/internalmodelbuilder.mlua",
["template.file"] = "template/file.lua",
["template.index"] = "template/index.lua",
["template.index.recordtypedef"] = "template/index/recordtypedef.lua",
["template.item"] = "template/item.lua",
["template.page"] = "template/page.lua",
["template.recordtypedef"] = "template/recordtypedef.lua",
["template.usage"] = "template/usage.lua",
["template.utils"] = "template/utils.lua",
templateengine = "templateengine.lua"
}
}
}
},
luafilesystem = {
["1.6.3-2"] = {
{
arch = "installed",
commands = {},
dependencies = {},
modules = {
lfs = "lfs.dll"
}
}
}
},
luasrcdiet = {
["0.3.0-2"] = {
{
arch = "installed",
commands = {
luasrcdiet = "luasrcdiet"
},
dependencies = {},
modules = {
luasrcdiet = "luasrcdiet/init.lua",
["luasrcdiet.equiv"] = "luasrcdiet/equiv.lua",
["luasrcdiet.fs"] = "luasrcdiet/fs.lua",
["luasrcdiet.llex"] = "luasrcdiet/llex.lua",
["luasrcdiet.lparser"] = "luasrcdiet/lparser.lua",
["luasrcdiet.optlex"] = "luasrcdiet/optlex.lua",
["luasrcdiet.optparser"] = "luasrcdiet/optparser.lua",
["luasrcdiet.plugin.example"] = "luasrcdiet/plugin/example.lua",
["luasrcdiet.plugin.html"] = "luasrcdiet/plugin/html.lua",
["luasrcdiet.plugin.sloc"] = "luasrcdiet/plugin/sloc.lua",
["luasrcdiet.utils"] = "luasrcdiet/utils.lua"
}
}
}
},
markdown = {
["0.32-2"] = {
{
arch = "installed",
commands = {},
dependencies = {},
modules = {
markdown = "markdown.lua"
}
}
}
},
["metalua-compiler"] = {
["0.7.3-1"] = {
{
arch = "installed",
commands = {},
dependencies = {
luafilesystem = "1.6.3-2",
["metalua-parser"] = "0.7.3-2"
},
modules = {
["metalua.compiler.bytecode"] = "metalua/compiler/bytecode.lua",
["metalua.compiler.bytecode.compile"] = "metalua/compiler/bytecode/compile.lua",
["metalua.compiler.bytecode.lcode"] = "metalua/compiler/bytecode/lcode.lua",
["metalua.compiler.bytecode.ldump"] = "metalua/compiler/bytecode/ldump.lua",
["metalua.compiler.bytecode.lopcodes"] = "metalua/compiler/bytecode/lopcodes.lua",
["metalua.compiler.globals"] = "metalua/compiler/globals.lua",
["metalua.loader"] = "metalua/loader.lua",
["metalua/compiler/ast_to_src.mlua"] = "metalua/compiler/ast_to_src.mlua",
["metalua/extension/comprehension.mlua"] = "metalua/extension/comprehension.mlua",
["metalua/extension/match.mlua"] = "metalua/extension/match.mlua",
["metalua/repl.mlua"] = "metalua/repl.mlua",
["metalua/treequery.mlua"] = "metalua/treequery.mlua",
["metalua/treequery/walk.mlua"] = "metalua/treequery/walk.mlua"
}
}
}
},
["metalua-parser"] = {
["0.7.3-2"] = {
{
arch = "installed",
commands = {},
dependencies = {},
modules = {
["metalua.compiler"] = "metalua/compiler.lua",
["metalua.compiler.parser"] = "metalua/compiler/parser.lua",
["metalua.compiler.parser.annot.generator"] = "metalua/compiler/parser/annot/generator.lua",
["metalua.compiler.parser.annot.grammar"] = "metalua/compiler/parser/annot/grammar.lua",
["metalua.compiler.parser.expr"] = "metalua/compiler/parser/expr.lua",
["metalua.compiler.parser.ext"] = "metalua/compiler/parser/ext.lua",
["metalua.compiler.parser.lexer"] = "metalua/compiler/parser/lexer.lua",
["metalua.compiler.parser.meta"] = "metalua/compiler/parser/meta.lua",
["metalua.compiler.parser.misc"] = "metalua/compiler/parser/misc.lua",
["metalua.compiler.parser.stat"] = "metalua/compiler/parser/stat.lua",
["metalua.compiler.parser.table"] = "metalua/compiler/parser/table.lua",
["metalua.grammar.generator"] = "metalua/grammar/generator.lua",
["metalua.grammar.lexer"] = "metalua/grammar/lexer.lua",
["metalua.pprint"] = "metalua/pprint.lua"
}
}
}
},
penlight = {
["0.9.8-1"] = {
{
arch = "installed",
commands = {},
dependencies = {
luafilesystem = "1.6.3-2"
},
modules = {
pl = "pl/init.lua",
["pl.Date"] = "pl/Date.lua",
["pl.List"] = "pl/List.lua",
["pl.Map"] = "pl/Map.lua",
["pl.MultiMap"] = "pl/MultiMap.lua",
["pl.OrderedMap"] = "pl/OrderedMap.lua",
["pl.Set"] = "pl/Set.lua",
["pl.app"] = "pl/app.lua",
["pl.array2d"] = "pl/array2d.lua",
["pl.class"] = "pl/class.lua",
["pl.comprehension"] = "pl/comprehension.lua",
["pl.config"] = "pl/config.lua",
["pl.data"] = "pl/data.lua",
["pl.dir"] = "pl/dir.lua",
["pl.file"] = "pl/file.lua",
["pl.func"] = "pl/func.lua",
["pl.input"] = "pl/input.lua",
["pl.lapp"] = "pl/lapp.lua",
["pl.lexer"] = "pl/lexer.lua",
["pl.luabalanced"] = "pl/luabalanced.lua",
["pl.operator"] = "pl/operator.lua",
["pl.path"] = "pl/path.lua",
["pl.permute"] = "pl/permute.lua",
["pl.platf.luajava"] = "pl/platf/luajava.lua",
["pl.pretty"] = "pl/pretty.lua",
["pl.seq"] = "pl/seq.lua",
["pl.sip"] = "pl/sip.lua",
["pl.strict"] = "pl/strict.lua",
["pl.stringio"] = "pl/stringio.lua",
["pl.stringx"] = "pl/stringx.lua",
["pl.tablex"] = "pl/tablex.lua",
["pl.template"] = "pl/template.lua",
["pl.test"] = "pl/test.lua",
["pl.text"] = "pl/text.lua",
["pl.utils"] = "pl/utils.lua",
["pl.xml"] = "pl/xml.lua"
}
}
}
}
}

View File

@@ -0,0 +1,23 @@
package = "Markdown"
version = "0.32-2"
source = {
url = "http://www.frykholm.se/files/markdown-0.32.tar.gz",
dir = "."
}
description = {
summary = "Markdown text-to-html markup system.",
detailed = [[
A pure-lua implementation of the Markdown text-to-html markup system.
]],
license = "MIT",
homepage = "http://www.frykholm.se/files/markdown.lua"
}
dependencies = {
"lua >= 5.1",
}
build = {
type = "none",
install = {
lua = { "markdown.lua" },
}
}

View File

@@ -0,0 +1,6 @@
rock_manifest = {
lua = {
["markdown.lua"] = "0ea5f9d6d22a6c9aa4fdf63cf1d7d066"
},
["markdown-0.32-2.rockspec"] = "83f0335058d8fbd078d4f2c1ce941df0"
}

View File

@@ -0,0 +1,104 @@
Metalua Compiler
================
## Metalua compiler
This module `metalua-compiler` depends on `metalua-parser`. Its main
feature is to compile ASTs into Lua 5.1 bytecode, allowing to convert
them into bytecode files and executable functions. This opens the
following possibilities:
* compiler objects generated with `require 'metalua.compiler'.new()`
support methods `:xxx_to_function()` and `:xxx_to_bytecode()`;
* Compile-time meta-programming: use of `-{...}` splices in source
code, to generate code during compilation;
* Some syntax extensions, such as structural pattern matching and
lists by comprehension;
* Some AST manipulation facilities such as `treequery`, which are
implemented with Metalua syntax extensions.
## What's new in Metalua 0.7
This is a major overhaul of the compiler's architecture. Some of the
most noteworthy changes are:
* No more installation or bootstrap script. Some Metalua source files
have been rewritten in plain Lua, and module sources have been
refactored, so that if you just drop the `metalua` folder somewhere
in your `LUA_PATH`, it works.
* The compiler can be cut in two parts:
* a parser which generates ASTs out of Lua sources, and should be
either portable or easily ported to Lua 5.2;
* a compiler, which can turn sources and AST into executable
Lua 5.1 bytecode and run it. It also supports compile-time
meta-programming, i.e. code included between `-{ ... }` is
executed during compilation, and the ASTs it produces are
included in the resulting bytecode.
* Both parts are packaged as separate LuaRocks, `metalua-parser` and
`metalua-compiler` respectively, so that you can install the former
without the latter.
* The parser is not a unique object anymore. Instead,
`require "metalua.compiler".new()` returns a different compiler
instance every time it's called. Compiler instances can be reused on
as many source files as wanted, but extending one instance's grammar
doesn't affect other compiler instances.
* Included standard library has been shed. There are too many standard
libs in Lua, and none of them is standard enough, offering
yet-another-one, coupled with a specific compiler can only add to
confusion.
* Many syntax extensions, which either were arguably more code samples
than actual production-ready tools, or relied too heavily on the
removed runtime standard libraries, have been removed.
* The remaining libraries and samples are:
* `metalua.compiler` converts sources into ASTs, bytecode,
functions, and ASTs back into sources.
* `metalua` compiles and/or executes files from the command line,
can start an interactive REPL session.
* `metalua.loader` adds a package loader which allows to use modules
written in Metalua, even from a plain Lua program.
* `metalua.treequery` is an advanced DSL allowing to search ASTs in
a smart way, e.g. "_search `return` statements which return a
`local` variable but aren't in a nested `function`_".
* `metalua.extension.comprehension` is a language extension which
supports lists by comprehension
(`even = { i for i=1, 100 if i%2==0 }`) and improved loops
(`for i=1, 10 for j=1,10 if i~=j do print(i,j) end`).
* `metalua.extension.match` is a language extension which offers
Haskell/ML structural pattern matching
(``match AST with `Function{ args, body } -> ... | `Number{ 0 } -> ...end``)
* **TODO Move basic extensions in a separate module.**
* To remove the compilation speed penalty associated with
metaprogramming, when environment variable `LUA_MCACHE` or Lua
variable `package.mcache` is defined and LuaFileSystem is available,
the results of Metalua source compilations is cached. Unless the
source file is more recent than the latest cached bytecode file, the
latter is loaded instead of the former.
* The Luarock install for the full compiler lists dependencies towards
Readline, LuaFileSytem, and Alt-Getopts. Those projects are
optional, but having them automatically installed by LuaRocks offers
a better user experience.
* The license has changed from MIT to double license MIT + EPL. This
has been done in order to provide the IP guarantees expected by the
Eclipse Foundation, to include Metalua in Eclipse's
[Lua Development Tools](http://www.eclipse.org/koneki/ldt/).

View File

@@ -0,0 +1,177 @@
Metalua Parser
==============
`metalua-parser` is a subset of the Metalua compiler, which turns
valid Lua source files and strings into abstract syntax trees
(AST). This README includes a description of this AST format. People
interested by Lua code analysis and generation are encouraged to
produce and/or consume this format to represent ASTs.
It has been designed for Lua 5.1. It hasn't been tested against
Lua 5.2, but should be easily ported.
## Usage
Module `metalua.compiler` has a `new()` function, which returns a
compiler instance. This instance has a set of methods of the form
`:xxx_to_yyy(input)`, where `xxx` and `yyy` must be one of the
following:
* `srcfile` the name of a Lua source file;
* `src` a string containing the Lua sources of a list of statements;
* `lexstream` a lexical tokens stream;
* `ast` an abstract syntax tree;
* `bytecode` a chunk of Lua bytecode that can be loaded in a Lua 5.1
VM (not available if you only installed the parser);
* `function` an executable Lua function.
Compiling into bytecode or executable functions requires the whole
Metalua compiler, not only the parser. The most frequently used
functions are `:src_to_ast(source_string)` and
`:srcfile_to_ast("path/to/source/file.lua")`.
mlc = require 'metalua.compiler'.new()
ast = mlc :src_to_ast[[ return 123 ]]
A compiler instance can be reused as much as you want; it's only
interesting to work with more than one compiler instance when you
start extending their grammars.
## Abstract Syntax Trees definition
### Notation
Trees are written below with some Metalua syntax sugar, which
increases their readability. the backquote symbol introduces a `tag`,
i.e. a string stored in the `"tag"` field of a table:
* `` `Foo{ 1, 2, 3 }`` is a shortcut for `{tag="Foo", 1, 2, 3}`;
* `` `Foo`` is a shortcut for `{tag="Foo"}`;
* `` `Foo 123`` is a shortcut for `` `Foo{ 123 }``, and therefore
`{tag="Foo", 123 }`; the expression after the tag must be a literal
number or string.
When using a Metalua interpreter or compiler, the backtick syntax is
supported and can be used directly. Metalua's pretty-printing helpers
also try to use backtick syntax whenever applicable.
### Tree elements
Tree elements are mainly categorized into statements `stat`,
expressions `expr` and lists of statements `block`. Auxiliary
definitions include function applications/method invocation `apply`,
are both valid statements and expressions, expressions admissible on
the left-hand-side of an assignment statement `lhs`.
block: { stat* }
stat:
`Do{ stat* }
| `Set{ {lhs+} {expr+} } -- lhs1, lhs2... = e1, e2...
| `While{ expr block } -- while e do b end
| `Repeat{ block expr } -- repeat b until e
| `If{ (expr block)+ block? } -- if e1 then b1 [elseif e2 then b2] ... [else bn] end
| `Fornum{ ident expr expr expr? block } -- for ident = e, e[, e] do b end
| `Forin{ {ident+} {expr+} block } -- for i1, i2... in e1, e2... do b end
| `Local{ {ident+} {expr+}? } -- local i1, i2... = e1, e2...
| `Localrec{ ident expr } -- only used for 'local function'
| `Goto{ <string> } -- goto str
| `Label{ <string> } -- ::str::
| `Return{ <expr*> } -- return e1, e2...
| `Break -- break
| apply
expr:
`Nil | `Dots | `True | `False
| `Number{ <number> }
| `String{ <string> }
| `Function{ { ident* `Dots? } block }
| `Table{ ( `Pair{ expr expr } | expr )* }
| `Op{ opid expr expr? }
| `Paren{ expr } -- significant to cut multiple values returns
| apply
| lhs
apply:
`Call{ expr expr* }
| `Invoke{ expr `String{ <string> } expr* }
ident: `Id{ <string> }
lhs: ident | `Index{ expr expr }
opid: 'add' | 'sub' | 'mul' | 'div'
| 'mod' | 'pow' | 'concat'| 'eq'
| 'lt' | 'le' | 'and' | 'or'
| 'not' | 'len'
### Meta-data (lineinfo)
ASTs also embed some metadata, allowing to map them to their source
representation. Those informations are stored in a `"lineinfo"` field
in each tree node, which points to the range of characters in the
source string which represents it, and to the content of any comment
that would appear immediately before or after that node.
Lineinfo objects have two fields, `"first"` and `"last"`, describing
respectively the beginning and the end of the subtree in the
sources. For instance, the sub-node ``Number{123}` produced by parsing
`[[return 123]]` will have `lineinfo.first` describing offset 8, and
`lineinfo.last` describing offset 10:
> mlc = require 'metalua.compiler'.new()
> ast = mlc :src_to_ast "return 123 -- comment"
> print(ast[1][1].lineinfo)
<?|L1|C8-10|K8-10|C>
>
A lineinfo keeps track of character offsets relative to the beginning
of the source string/file ("K8-10" above), line numbers (L1 above; a
lineinfo spanning on several lines would read something like "L1-10"),
columns i.e. offset within the line ("C8-10" above), and a filename if
available (the "?" mark above indicating that we have no file name, as
the AST comes from a string). The final "|C>" indicates that there's a
comment immediately after the node; an initial "<C|" would have meant
that there was a comment immediately before the node.
Positions represent either the end of a token and the beginning of an
inter-token space (`"last"` fields) or the beginning of a token, and
the end of an inter-token space (`"first"` fields). Inter-token spaces
might be empty. They can also contain comments, which might be useful
to link with surrounding tokens and AST subtrees.
Positions are chained with their "dual" one: a position at the
beginning of and inter-token space keeps a refernce to the position at
the end of that inter-token space in its `"facing"` field, and
conversly, end-of-inter-token positions keep track of the inter-token
space beginning, also in `"facing"`. An inter-token space can be
empty, e.g. in `"2+2"`, in which case `lineinfo==lineinfo.facing`.
Comments are also kept in the `"comments"` field. If present, this
field contains a list of comments, with a `"lineinfo"` field
describing the span between the first and last comment. Each comment
is represented by a list of one string, with a `"lineinfo"` describing
the span of this comment only. Consecutive lines of `--` comments are
considered as one comment: `"-- foo\n-- bar\n"` parses as one comment
whose text is `"foo\nbar"`, whereas `"-- foo\n\n-- bar\n"` parses as
two comments `"foo"` and `"bar"`.
So for instance, if `f` is the AST of a function and I want to
retrieve the comment before the function, I'd do:
f_comment = f.lineinfo.first.comments[1][1]
The informations in lineinfo positions, i.e. in each `"first"` and
`"last"` field, are held in the following fields:
* `"source"` the filename (optional);
* `"offset"` the 1-based offset relative to the beginning of the string/file;
* `"line"` the 1-based line number;
* `"column"` the 1-based offset within the line;
* `"facing"` the position at the opposite end of the inter-token space.
* `"comments"` the comments in the associated inter-token space (optional).
* `"id"` an arbitrary number, which uniquely identifies an inter-token
space within a given tokens stream.

View File

@@ -0,0 +1,13 @@
Metalua
=======
Metalua is a Lua code analysis tool, as well as a compiler for a
superset of Lua 5.1 supporting Compile-Time Meta-Programming. It's
separated into two LuaRocks, `metalua-parser` and
`metalua-compiler`. The documentation of each rock can be found in
`README-parser.md` and `README-compiler.md`.
All the code in Metalue is released under dual lincenses:
* MIT public license (same as Lua);
* EPL public license (same as Eclipse).

View File

@@ -0,0 +1,47 @@
--*-lua-*--
package = "metalua-compiler"
version = "0.7.3-1"
source = {
url = "http://git.eclipse.org/c/koneki/org.eclipse.koneki.metalua.git/snapshot/org.eclipse.koneki.metalua-v0.7.3.tar.gz"
}
description = {
summary = "Metalua's compiler: converting (Meta)lua source strings and files into executable Lua 5.1 bytecode",
detailed = [[
This is the Metalua copmiler, packaged as a rock, depending
on the spearate metalua-parser AST generating library. It
compiles a superset of Lua 5.1 into bytecode, which can
then be loaded and executed by a Lua 5.1 VM. It also allows
to dump ASTs back into Lua source files.
]],
homepage = "http://git.eclipse.org/c/koneki/org.eclipse.koneki.metalua.git",
license = "EPL + MIT"
}
dependencies = {
"lua ~> 5.1", -- Lua 5.2 bytecode not supported
"luafilesystem ~> 1.6", -- Cached compilation based on file timestamps
"metalua-parser >= 0.7.3", -- AST production
}
build = {
type="builtin",
modules={
["metalua.compiler.bytecode"] = "metalua/compiler/bytecode.lua",
["metalua.compiler.globals"] = "metalua/compiler/globals.lua",
["metalua.compiler.bytecode.compile"] = "metalua/compiler/bytecode/compile.lua",
["metalua.compiler.bytecode.lcode"] = "metalua/compiler/bytecode/lcode.lua",
["metalua.compiler.bytecode.lopcodes"] = "metalua/compiler/bytecode/lopcodes.lua",
["metalua.compiler.bytecode.ldump"] = "metalua/compiler/bytecode/ldump.lua",
["metalua.loader"] = "metalua/loader.lua",
},
install={
lua={
["metalua.treequery"] = "metalua/treequery.mlua",
["metalua.compiler.ast_to_src"] = "metalua/compiler/ast_to_src.mlua",
["metalua.treequery.walk"] = "metalua/treequery/walk.mlua",
["metalua.extension.match"] = "metalua/extension/match.mlua",
["metalua.extension.comprehension"] = "metalua/extension/comprehension.mlua",
["metalua.repl"] = "metalua/repl.mlua",
}
}
}

View File

@@ -0,0 +1,33 @@
rock_manifest = {
doc = {
["README-compiler.md"] = "292523d759247d210d32fb2f6153e0f4",
["README-parser.md"] = "b44e3673d96dd296f2c0e92a6c87ee18",
["README.md"] = "20bfb490cddef9e101e44688791abcda"
},
lua = {
metalua = {
compiler = {
["ast_to_src.mlua"] = "1309f76df37585ef8e1f67f748b07b22",
bytecode = {
["compile.lua"] = "430e4a6fac8b64b5ebb3ae585ebae75a",
["lcode.lua"] = "3ad8755ebe8ea8eca6b1d2846eec92c4",
["ldump.lua"] = "295e1d9657fb0126ce3471b3366da694",
["lopcodes.lua"] = "a0f15cfc93b026b0a868466d066f1d21"
},
["bytecode.lua"] = "1032e5233455fd4e504daf5d2893527b",
["globals.lua"] = "80ae19c6e640de0746348c91633c4c55"
},
extension = {
["comprehension.mlua"] = "426f5856896bda4c3763bd5f61410685",
["match.mlua"] = "79960265331e8b2f46199c2411a103de"
},
["loader.lua"] = "1cdbf6cdf6ca97c55540d068474f1d8a",
["repl.mlua"] = "729456f3a8cc073788acee564a0495f0",
treequery = {
["walk.mlua"] = "5159aaddbec55936f91ea4236f6451d3"
},
["treequery.mlua"] = "97ffcee0825ac3bc776d01566767b2e8"
}
},
["metalua-compiler-0.7.3-1.rockspec"] = "b3883b25641d862db6828300bb755d51"
}

View File

@@ -0,0 +1,104 @@
Metalua Compiler
================
## Metalua compiler
This module `metalua-compiler` depends on `metalua-parser`. Its main
feature is to compile ASTs into Lua 5.1 bytecode, allowing to convert
them into bytecode files and executable functions. This opens the
following possibilities:
* compiler objects generated with `require 'metalua.compiler'.new()`
support methods `:xxx_to_function()` and `:xxx_to_bytecode()`;
* Compile-time meta-programming: use of `-{...}` splices in source
code, to generate code during compilation;
* Some syntax extensions, such as structural pattern matching and
lists by comprehension;
* Some AST manipulation facilities such as `treequery`, which are
implemented with Metalua syntax extensions.
## What's new in Metalua 0.7
This is a major overhaul of the compiler's architecture. Some of the
most noteworthy changes are:
* No more installation or bootstrap script. Some Metalua source files
have been rewritten in plain Lua, and module sources have been
refactored, so that if you just drop the `metalua` folder somewhere
in your `LUA_PATH`, it works.
* The compiler can be cut in two parts:
* a parser which generates ASTs out of Lua sources, and should be
either portable or easily ported to Lua 5.2;
* a compiler, which can turn sources and AST into executable
Lua 5.1 bytecode and run it. It also supports compile-time
meta-programming, i.e. code included between `-{ ... }` is
executed during compilation, and the ASTs it produces are
included in the resulting bytecode.
* Both parts are packaged as separate LuaRocks, `metalua-parser` and
`metalua-compiler` respectively, so that you can install the former
without the latter.
* The parser is not a unique object anymore. Instead,
`require "metalua.compiler".new()` returns a different compiler
instance every time it's called. Compiler instances can be reused on
as many source files as wanted, but extending one instance's grammar
doesn't affect other compiler instances.
* Included standard library has been shed. There are too many standard
libs in Lua, and none of them is standard enough, offering
yet-another-one, coupled with a specific compiler can only add to
confusion.
* Many syntax extensions, which either were arguably more code samples
than actual production-ready tools, or relied too heavily on the
removed runtime standard libraries, have been removed.
* The remaining libraries and samples are:
* `metalua.compiler` converts sources into ASTs, bytecode,
functions, and ASTs back into sources.
* `metalua` compiles and/or executes files from the command line,
can start an interactive REPL session.
* `metalua.loader` adds a package loader which allows to use modules
written in Metalua, even from a plain Lua program.
* `metalua.treequery` is an advanced DSL allowing to search ASTs in
a smart way, e.g. "_search `return` statements which return a
`local` variable but aren't in a nested `function`_".
* `metalua.extension.comprehension` is a language extension which
supports lists by comprehension
(`even = { i for i=1, 100 if i%2==0 }`) and improved loops
(`for i=1, 10 for j=1,10 if i~=j do print(i,j) end`).
* `metalua.extension.match` is a language extension which offers
Haskell/ML structural pattern matching
(``match AST with `Function{ args, body } -> ... | `Number{ 0 } -> ...end``)
* **TODO Move basic extensions in a separate module.**
* To remove the compilation speed penalty associated with
metaprogramming, when environment variable `LUA_MCACHE` or Lua
variable `package.mcache` is defined and LuaFileSystem is available,
the results of Metalua source compilations is cached. Unless the
source file is more recent than the latest cached bytecode file, the
latter is loaded instead of the former.
* The Luarock install for the full compiler lists dependencies towards
Readline, LuaFileSytem, and Alt-Getopts. Those projects are
optional, but having them automatically installed by LuaRocks offers
a better user experience.
* The license has changed from MIT to double license MIT + EPL. This
has been done in order to provide the IP guarantees expected by the
Eclipse Foundation, to include Metalua in Eclipse's
[Lua Development Tools](http://www.eclipse.org/koneki/ldt/).

View File

@@ -0,0 +1,177 @@
Metalua Parser
==============
`metalua-parser` is a subset of the Metalua compiler, which turns
valid Lua source files and strings into abstract syntax trees
(AST). This README includes a description of this AST format. People
interested by Lua code analysis and generation are encouraged to
produce and/or consume this format to represent ASTs.
It has been designed for Lua 5.1. It hasn't been tested against
Lua 5.2, but should be easily ported.
## Usage
Module `metalua.compiler` has a `new()` function, which returns a
compiler instance. This instance has a set of methods of the form
`:xxx_to_yyy(input)`, where `xxx` and `yyy` must be one of the
following:
* `srcfile` the name of a Lua source file;
* `src` a string containing the Lua sources of a list of statements;
* `lexstream` a lexical tokens stream;
* `ast` an abstract syntax tree;
* `bytecode` a chunk of Lua bytecode that can be loaded in a Lua 5.1
VM (not available if you only installed the parser);
* `function` an executable Lua function.
Compiling into bytecode or executable functions requires the whole
Metalua compiler, not only the parser. The most frequently used
functions are `:src_to_ast(source_string)` and
`:srcfile_to_ast("path/to/source/file.lua")`.
mlc = require 'metalua.compiler'.new()
ast = mlc :src_to_ast[[ return 123 ]]
A compiler instance can be reused as much as you want; it's only
interesting to work with more than one compiler instance when you
start extending their grammars.
## Abstract Syntax Trees definition
### Notation
Trees are written below with some Metalua syntax sugar, which
increases their readability. the backquote symbol introduces a `tag`,
i.e. a string stored in the `"tag"` field of a table:
* `` `Foo{ 1, 2, 3 }`` is a shortcut for `{tag="Foo", 1, 2, 3}`;
* `` `Foo`` is a shortcut for `{tag="Foo"}`;
* `` `Foo 123`` is a shortcut for `` `Foo{ 123 }``, and therefore
`{tag="Foo", 123 }`; the expression after the tag must be a literal
number or string.
When using a Metalua interpreter or compiler, the backtick syntax is
supported and can be used directly. Metalua's pretty-printing helpers
also try to use backtick syntax whenever applicable.
### Tree elements
Tree elements are mainly categorized into statements `stat`,
expressions `expr` and lists of statements `block`. Auxiliary
definitions include function applications/method invocation `apply`,
are both valid statements and expressions, expressions admissible on
the left-hand-side of an assignment statement `lhs`.
block: { stat* }
stat:
`Do{ stat* }
| `Set{ {lhs+} {expr+} } -- lhs1, lhs2... = e1, e2...
| `While{ expr block } -- while e do b end
| `Repeat{ block expr } -- repeat b until e
| `If{ (expr block)+ block? } -- if e1 then b1 [elseif e2 then b2] ... [else bn] end
| `Fornum{ ident expr expr expr? block } -- for ident = e, e[, e] do b end
| `Forin{ {ident+} {expr+} block } -- for i1, i2... in e1, e2... do b end
| `Local{ {ident+} {expr+}? } -- local i1, i2... = e1, e2...
| `Localrec{ ident expr } -- only used for 'local function'
| `Goto{ <string> } -- goto str
| `Label{ <string> } -- ::str::
| `Return{ <expr*> } -- return e1, e2...
| `Break -- break
| apply
expr:
`Nil | `Dots | `True | `False
| `Number{ <number> }
| `String{ <string> }
| `Function{ { ident* `Dots? } block }
| `Table{ ( `Pair{ expr expr } | expr )* }
| `Op{ opid expr expr? }
| `Paren{ expr } -- significant to cut multiple values returns
| apply
| lhs
apply:
`Call{ expr expr* }
| `Invoke{ expr `String{ <string> } expr* }
ident: `Id{ <string> }
lhs: ident | `Index{ expr expr }
opid: 'add' | 'sub' | 'mul' | 'div'
| 'mod' | 'pow' | 'concat'| 'eq'
| 'lt' | 'le' | 'and' | 'or'
| 'not' | 'len'
### Meta-data (lineinfo)
ASTs also embed some metadata, allowing to map them to their source
representation. Those informations are stored in a `"lineinfo"` field
in each tree node, which points to the range of characters in the
source string which represents it, and to the content of any comment
that would appear immediately before or after that node.
Lineinfo objects have two fields, `"first"` and `"last"`, describing
respectively the beginning and the end of the subtree in the
sources. For instance, the sub-node ``Number{123}` produced by parsing
`[[return 123]]` will have `lineinfo.first` describing offset 8, and
`lineinfo.last` describing offset 10:
> mlc = require 'metalua.compiler'.new()
> ast = mlc :src_to_ast "return 123 -- comment"
> print(ast[1][1].lineinfo)
<?|L1|C8-10|K8-10|C>
>
A lineinfo keeps track of character offsets relative to the beginning
of the source string/file ("K8-10" above), line numbers (L1 above; a
lineinfo spanning on several lines would read something like "L1-10"),
columns i.e. offset within the line ("C8-10" above), and a filename if
available (the "?" mark above indicating that we have no file name, as
the AST comes from a string). The final "|C>" indicates that there's a
comment immediately after the node; an initial "<C|" would have meant
that there was a comment immediately before the node.
Positions represent either the end of a token and the beginning of an
inter-token space (`"last"` fields) or the beginning of a token, and
the end of an inter-token space (`"first"` fields). Inter-token spaces
might be empty. They can also contain comments, which might be useful
to link with surrounding tokens and AST subtrees.
Positions are chained with their "dual" one: a position at the
beginning of and inter-token space keeps a refernce to the position at
the end of that inter-token space in its `"facing"` field, and
conversly, end-of-inter-token positions keep track of the inter-token
space beginning, also in `"facing"`. An inter-token space can be
empty, e.g. in `"2+2"`, in which case `lineinfo==lineinfo.facing`.
Comments are also kept in the `"comments"` field. If present, this
field contains a list of comments, with a `"lineinfo"` field
describing the span between the first and last comment. Each comment
is represented by a list of one string, with a `"lineinfo"` describing
the span of this comment only. Consecutive lines of `--` comments are
considered as one comment: `"-- foo\n-- bar\n"` parses as one comment
whose text is `"foo\nbar"`, whereas `"-- foo\n\n-- bar\n"` parses as
two comments `"foo"` and `"bar"`.
So for instance, if `f` is the AST of a function and I want to
retrieve the comment before the function, I'd do:
f_comment = f.lineinfo.first.comments[1][1]
The informations in lineinfo positions, i.e. in each `"first"` and
`"last"` field, are held in the following fields:
* `"source"` the filename (optional);
* `"offset"` the 1-based offset relative to the beginning of the string/file;
* `"line"` the 1-based line number;
* `"column"` the 1-based offset within the line;
* `"facing"` the position at the opposite end of the inter-token space.
* `"comments"` the comments in the associated inter-token space (optional).
* `"id"` an arbitrary number, which uniquely identifies an inter-token
space within a given tokens stream.

View File

@@ -0,0 +1,13 @@
Metalua
=======
Metalua is a Lua code analysis tool, as well as a compiler for a
superset of Lua 5.1 supporting Compile-Time Meta-Programming. It's
separated into two LuaRocks, `metalua-parser` and
`metalua-compiler`. The documentation of each rock can be found in
`README-parser.md` and `README-compiler.md`.
All the code in Metalue is released under dual lincenses:
* MIT public license (same as Lua);
* EPL public license (same as Eclipse).

View File

@@ -0,0 +1,38 @@
--*-lua-*--
package = "metalua-parser"
version = "0.7.3-2"
source = {
url = "http://git.eclipse.org/c/koneki/org.eclipse.koneki.metalua.git/snapshot/org.eclipse.koneki.metalua-v0.7.3.tar.gz"
}
description = {
summary = "Metalua's parser: converting Lua source strings and files into AST",
detailed = [[
This is a subset of the full Metalua compiler. It defines and generates an AST
format for Lua programs, which offers a nice level of abstraction to reason about
and manipulate Lua programs.
]],
homepage = "http://git.eclipse.org/c/koneki/org.eclipse.koneki.metalua.git",
license = "EPL + MIT"
}
dependencies = {
"lua >= 5.1"
}
build = {
type="builtin",
modules={
["metalua.grammar.generator"] = "metalua/grammar/generator.lua",
["metalua.grammar.lexer"] = "metalua/grammar/lexer.lua",
["metalua.compiler.parser"] = "metalua/compiler/parser.lua",
["metalua.compiler.parser.table"] = "metalua/compiler/parser/table.lua",
["metalua.compiler.parser.ext"] = "metalua/compiler/parser/ext.lua",
["metalua.compiler.parser.annot.generator"] = "metalua/compiler/parser/annot/generator.lua",
["metalua.compiler.parser.annot.grammar"] = "metalua/compiler/parser/annot/grammar.lua",
["metalua.compiler.parser.stat"] = "metalua/compiler/parser/stat.lua",
["metalua.compiler.parser.misc"] = "metalua/compiler/parser/misc.lua",
["metalua.compiler.parser.lexer"] = "metalua/compiler/parser/lexer.lua",
["metalua.compiler.parser.meta"] = "metalua/compiler/parser/meta.lua",
["metalua.compiler.parser.expr"] = "metalua/compiler/parser/expr.lua",
["metalua.compiler"] = "metalua/compiler.lua",
["metalua.pprint"] = "metalua/pprint.lua",
}
}

View File

@@ -0,0 +1,34 @@
rock_manifest = {
doc = {
["README-compiler.md"] = "292523d759247d210d32fb2f6153e0f4",
["README-parser.md"] = "b44e3673d96dd296f2c0e92a6c87ee18",
["README.md"] = "20bfb490cddef9e101e44688791abcda"
},
lua = {
metalua = {
compiler = {
parser = {
annot = {
["generator.lua"] = "d86f7507d66ba6a3692a6f8611e9939b",
["grammar.lua"] = "7d195bde7992efd9923771751b67b18f"
},
["expr.lua"] = "3a0b1984a6f92280e2e63b074fdcec10",
["ext.lua"] = "a99e31a07bc390b826f6653bcc47d89b",
["lexer.lua"] = "eac0f9d475d9dae4ea5a2724014cebec",
["meta.lua"] = "12870bceda6395695020b739196e2a92",
["misc.lua"] = "49d59f4fc1bfb77b36f78d4f87ae258f",
["stat.lua"] = "83f10ac899be12ca4df58bbe8645299f",
["table.lua"] = "5d2389e89603b7f78c731e6918aa1a9b"
},
["parser.lua"] = "e6ae68ce200de8071bb0fefad97f9b79"
},
["compiler.lua"] = "ca65ee9a3053581f4315821a31d0c1fd",
grammar = {
["generator.lua"] = "b8a29e817d6798c12f40a230a0f6d0af",
["lexer.lua"] = "7cb7c835479a9be884130eaacb9be60a"
},
["pprint.lua"] = "0b9bd8757b45c2d4be30106abcbd45b2"
}
},
["metalua-parser-0.7.3-2.rockspec"] = "a56680900b0b51701db7cd7abf49af92"
}

View File

@@ -0,0 +1,66 @@
package = "penlight"
version = "0.9.8-1"
source = {
dir = "penlight-0.9.8",
url = "http://stevedonovan.github.com/files/penlight-0.9.8-core.zip",
}
description = {
summary = "Lua utility libraries loosely based on the Python standard libraries",
homepage = "http://stevedonovan.github.com/Penlight",
license = "MIT/X11",
maintainer = "steve.j.donovan@gmail.com",
detailed = [[
Penlight is a set of pure Lua libraries for making it easier to work with common tasks like
iterating over directories, reading configuration files and the like. Provides functional operations
on tables and sequences.
]]
}
dependencies = {
"luafilesystem",
}
build = {
type = "builtin",
modules = {
["pl.strict"] = "lua/pl/strict.lua",
["pl.dir"] = "lua/pl/dir.lua",
["pl.operator"] = "lua/pl/operator.lua",
["pl.input"] = "lua/pl/input.lua",
["pl.config"] = "lua/pl/config.lua",
["pl.seq"] = "lua/pl/seq.lua",
["pl.stringio"] = "lua/pl/stringio.lua",
["pl.text"] = "lua/pl/text.lua",
["pl.test"] = "lua/pl/test.lua",
["pl.tablex"] = "lua/pl/tablex.lua",
["pl.app"] = "lua/pl/app.lua",
["pl.stringx"] = "lua/pl/stringx.lua",
["pl.lexer"] = "lua/pl/lexer.lua",
["pl.utils"] = "lua/pl/utils.lua",
["pl.sip"] = "lua/pl/sip.lua",
["pl.permute"] = "lua/pl/permute.lua",
["pl.pretty"] = "lua/pl/pretty.lua",
["pl.class"] = "lua/pl/class.lua",
["pl.List"] = "lua/pl/List.lua",
["pl.data"] = "lua/pl/data.lua",
["pl.Date"] = "lua/pl/Date.lua",
["pl"] = "lua/pl/init.lua",
["pl.luabalanced"] = "lua/pl/luabalanced.lua",
["pl.comprehension"] = "lua/pl/comprehension.lua",
["pl.path"] = "lua/pl/path.lua",
["pl.array2d"] = "lua/pl/array2d.lua",
["pl.func"] = "lua/pl/func.lua",
["pl.lapp"] = "lua/pl/lapp.lua",
["pl.file"] = "lua/pl/file.lua",
['pl.template'] = "lua/pl/template.lua",
["pl.Map"] = "lua/pl/Map.lua",
["pl.MultiMap"] = "lua/pl/MultiMap.lua",
["pl.OrderedMap"] = "lua/pl/OrderedMap.lua",
["pl.Set"] = "lua/pl/Set.lua",
["pl.xml"] = "lua/pl/xml.lua",
["pl.platf.luajava"] = "lua/pl/platf/luajava.lua"
},
}

View File

@@ -0,0 +1,45 @@
rock_manifest = {
lua = {
pl = {
["Date.lua"] = "d2131d59151ce978c4db6a648fcd275a",
["List.lua"] = "1236c5eb08956619daacd25a462a9682",
["Map.lua"] = "0297a536ac0595ac59e8828f8c867f53",
["MultiMap.lua"] = "e5f898fe2443e51c38825e9bc3d1aee5",
["OrderedMap.lua"] = "bd8e39c59e22c582a33e2f025d3ae914",
["Set.lua"] = "346ff7392fd4aeda418fb832e8da7a7f",
["app.lua"] = "23ffb79e69a3fd679013cf82d95ed792",
["array2d.lua"] = "77618ec2e2de4d6d237484dfd742cd73",
["class.lua"] = "6f58bf39e7f90711b6840ad6955d258e",
["comprehension.lua"] = "f8600ba945dde5d959194500a687c69f",
["config.lua"] = "9ea3ce0ac3cdf2ce0e17f1353f32abb6",
["data.lua"] = "be446ff813b5bcf30b4063601165df6a",
["dir.lua"] = "3d60d4c1caeaabe199fe361e4e9b14a4",
["file.lua"] = "f5c9527ea14b511d2cb9af80b219c562",
["func.lua"] = "cc50d73512b6d0518f6587b82844de8c",
["init.lua"] = "9232be7d8790d4f907972a00dec7949d",
["input.lua"] = "bab7c64ca9a740df5e9fb9909610bbc4",
["lapp.lua"] = "1cc81f048bc3fcd775c40cd9a2d601a7",
["lexer.lua"] = "da0db5e323a2d37545ccb02592d0d3c8",
["luabalanced.lua"] = "00b94a997a9ea4d73f54c10893f3b35f",
["operator.lua"] = "e606629c738966cf497bb938457adebd",
["path.lua"] = "b0714bc337c068b7252f64250fe59604",
["permute.lua"] = "b0ed9ba2787119ef99468329a54ea16a",
platf = {
["luajava.lua"] = "9c2898667281ad9501cc05a8e31a6f53"
},
["pretty.lua"] = "3ece64317ce05916eaba91fa96d9e7c0",
["seq.lua"] = "e99e420345ab11120a7b741d8184920a",
["sip.lua"] = "bde74f65e7246017d3ef034d178100ea",
["strict.lua"] = "720e939931dbbe42fad8fd4e7736435e",
["stringio.lua"] = "a8f4c786ea1b62f16ed05e6b09840044",
["stringx.lua"] = "43f57755969c6b4001316226506a3744",
["tablex.lua"] = "dec027cc3a3901766bd933c5fc0f3e93",
["template.lua"] = "f358175bbb84c401c6213c953ce295a4",
["test.lua"] = "1c45f7b1c438673f1eb668e2ca592f1c",
["text.lua"] = "c30f90cab2d00186a6432e408ba1fe14",
["utils.lua"] = "68cd38638a29b4ab5f1cc0eae38dce77",
["xml.lua"] = "e13ed468c450fccb9a8e858a0f787eef"
}
},
["penlight-0.9.8-1.rockspec"] = "96edac3ff1d0ac57cb45d6551a56a775"
}