Do not include files that have already been included.

If two #included files have the same inode, only the first include will
be parsed. this obsoletes the "#if ... #endinput" hack in every include
file.

Nominally #include in SourcePawn was the same as C/C++, where you can
include the same file over and over. This is useful in C/C++ for
creating macro expansion tables. It's not useful in Pawn, where those
tricks don't work, and as evidenced by the fact that not a single plugin
in the corpus was affected by this change.

This greatly simplifies how static scopes are handled, since
SourcePawn's notion of a static scope is per-file, rather than
per-translation unit. This simplification re-opens the door to multiple
translation units, an important step for an upcoming refactoring of how
builtins work.

compiler/compile-context.h

@@ -62,7 +62,7 @@ class CompileContext final
     void TrackMalloc(size_t bytes);
     void TrackFree(size_t bytes);
 
-    sp::SymbolScope* globals() const { return globals_; }
+    SymbolScope* globals() const { return globals_; }
     tr::unordered_set<symbol*>& functions() { return functions_; }
     tr::unordered_set<symbol*>& publics() { return publics_; }
     const std::shared_ptr<Lexer>& lexer() const { return lexer_; }
@@ -104,8 +104,8 @@ class CompileContext final
     std::string& outfname() { return outfname_; }
     void set_outfname(const std::string& value) { outfname_ = value; }
 
-    std::shared_ptr<sp::SourceFile> inpf_org() const { return inpf_org_; }
-    void set_inpf_org(std::shared_ptr<sp::SourceFile> sf) { inpf_org_ = sf; }
+    std::shared_ptr<SourceFile> inpf_org() const { return inpf_org_; }
+    void set_inpf_org(std::shared_ptr<SourceFile> sf) { inpf_org_ = sf; }
 
     bool must_abort() const { return must_abort_; }
     void set_must_abort() { must_abort_ = true; }
@@ -130,17 +130,17 @@ class CompileContext final
 
   private:
     cc::PoolAllocator allocator_;
-    sp::SymbolScope* globals_;
+    SymbolScope* globals_;
     std::string default_include_;
     tr::unordered_set<symbol*> functions_;
     tr::unordered_set<symbol*> publics_;
     std::unique_ptr<CompileOptions> options_;
     std::string outfname_;
     std::string errfname_;
     std::unique_ptr<SourceManager> sources_;
-    std::shared_ptr<sp::SourceFile> inpf_org_;
+    std::shared_ptr<SourceFile> inpf_org_;
     std::unique_ptr<TypeDictionary> types_;
-    sp::StringPool atoms_;
+    StringPool atoms_;
 
     // The lexer is in CompileContext rather than Parser until we can eliminate
     // PreprocExpr().

compiler/lexer.cpp

@@ -77,6 +77,8 @@ bool Lexer::PlungeQualifiedFile(const std::string& name) {
     auto fp = OpenFile(name);
     if (!fp)
         return false;
+    if (fp->included())
+        return true;
 
     assert(!IsSkipping());
     assert(skiplevel_ == ifstack_.size()); /* these two are always the same when "parsing" */
@@ -2313,6 +2315,8 @@ void Lexer::EnterFile(std::shared_ptr<SourceFile>&& sf, const token_pos_t& from)
     SkipUtf8Bom();
     SetFileDefines(state_.inpf->name());
 
+    state_.inpf->set_included();
+
     tokens_on_line_ = 0;
 }
 

compiler/parser.cpp

@@ -59,7 +59,11 @@ Parser::Parse()
     });
 
     std::vector<Stmt*> stmts;
-    CreateInitialScopes(&stmts);
+
+    if (!cc_.default_include().empty()) {
+        auto incfname = cc_.default_include();
+        lexer_->PlungeFile(incfname, FALSE, TRUE);
+    }
 
     // Prime the lexer.
     lexer_->Start();
@@ -70,20 +74,9 @@ Parser::Parse()
 
         int tok = lexer_->lex();
 
-        // We don't have end-of-file tokens (yet), so we pop static scopes
-        // before every declaration. This should be after lexer_->lex() so we've
-        // processed any end-of-file events. 
-        bool changed = false;
-        int fcurrent = lexer_->fcurrent();
-        while (!static_scopes_.empty() && static_scopes_.back()->fnumber() != fcurrent) {
-            changed = true;
-            static_scopes_.pop_back();
-        }
-        assert(!static_scopes_.empty());
-
-        if (changed) {
-            stmts.emplace_back(new ChangeScopeNode(lexer_->pos(), static_scopes_.back(),
-                                                   lexer_->inpf()->name()));
+        if (sources_index_ != lexer_->fcurrent()) {
+            ChangeStaticScope(&stmts);
+            sources_index_ = lexer_->fcurrent();
         }
 
         switch (tok) {
@@ -156,10 +149,6 @@ Parser::Parse()
                     report(417) << name.substr(1);
                     cc_.set_must_abort();
                 }
-
-                int fcurrent = lexer_->fcurrent();
-                static_scopes_.emplace_back(new SymbolScope(cc_.globals(), sFILE_STATIC, fcurrent));
-                decl = new ChangeScopeNode(lexer_->pos(), static_scopes_.back(), name.substr(1));
                 break;
             }
             case '}':
@@ -212,25 +201,15 @@ Parser::Parse()
     return new ParseTree(list);
 }
 
-void Parser::CreateInitialScopes(std::vector<Stmt*>* stmts) {
-    // Create a static scope for the main file.
-    {
-        int fcurrent = lexer_->fcurrent();
-        assert(fcurrent == 0);
-        static_scopes_.emplace_back(new SymbolScope(cc_.globals(), sFILE_STATIC, fcurrent));
-        stmts->emplace_back(new ChangeScopeNode({}, static_scopes_.back(),
-                                                lexer_->inpf()->name()));
+void Parser::ChangeStaticScope(std::vector<Stmt*>* stmts) {
+    auto sources_index = lexer_->fcurrent();
+    auto iter = static_scopes_.find(sources_index);
+    if (iter == static_scopes_.end()) {
+        auto scope = new SymbolScope(cc_.globals(), sFILE_STATIC, sources_index);
+        iter = static_scopes_.emplace(sources_index, scope).first;
     }
 
-    if (!cc_.default_include().empty()) {
-        auto incfname = cc_.default_include();
-        if (lexer_->PlungeFile(incfname, FALSE, TRUE)) {
-            int fcurrent = lexer_->fcurrent();
-            static_scopes_.emplace_back(new SymbolScope(cc_.globals(), sFILE_STATIC, fcurrent));
-            stmts->emplace_back(new ChangeScopeNode({}, static_scopes_.back(),
-                                lexer_->inpf()->name()));
-        }
-    }
+    stmts->emplace_back(new ChangeScopeNode(lexer_->pos(), iter->second,                        lexer_->inpf()->name()));
 }
 
 Stmt*

compiler/parser.h

@@ -25,6 +25,7 @@
 #include "parse-node.h"
 #include "sc.h"
 #include "sctracker.h"
+#include "stl/stl-deque.h"
 
 namespace sp {
 
@@ -44,8 +45,10 @@ class Parser
     typedef int (Parser::*HierFn)(value*);
     typedef Expr* (Parser::*NewHierFn)();
 
+
     static symbol* ParseInlineFunction(int tokid, const declinfo_t& decl, const int* this_tag);
-    void CreateInitialScopes(std::vector<Stmt*>* list);
+
+    void ChangeStaticScope(std::vector<Stmt*>* stmts);
 
     Stmt* parse_unknown_decl(const full_token_t* tok);
     Decl* parse_enum(int vclass);
@@ -135,10 +138,11 @@ class Parser
     Semantics* sema_;
     bool in_loop_ = false;
     bool in_test_ = false;
-    std::vector<sp::SymbolScope*> static_scopes_;
     std::shared_ptr<Lexer> lexer_;
     TypeDictionary* types_ = nullptr;
-    std::deque<FunctionDecl*> delayed_functions_;
+    tr::deque<FunctionDecl*> delayed_functions_;
+    tr::unordered_map<size_t, SymbolScope*> static_scopes_;
+    int sources_index_ = -1;
 };
 
 } // namespace sp

compiler/source-file.h

@@ -52,6 +52,9 @@ class SourceFile : public std::enable_shared_from_this<SourceFile>
     bool is_main_file() const { return is_main_file_; }
     void set_is_main_file() { is_main_file_ = true; }
 
+    bool included() const { return included_; }
+    void set_included() { included_ = true; }
+
     void operator =(const SourceFile&) = delete;
     void operator =(SourceFile&&) = delete;
     const unsigned char* data() const {
@@ -75,6 +78,7 @@ class SourceFile : public std::enable_shared_from_this<SourceFile>
     tr::string data_;
     size_t pos_;
     bool is_main_file_ = false;
+    bool included_ = false;
     ke::Maybe<uint32_t> sources_index_;
     tr::vector<uint32_t> line_extents_;
 };

compiler/stl/stl-deque.h

@@ -0,0 +1,31 @@
+// vim: set sts=4 ts=8 sw=4 tw=99 et:
+//
+// Copyright (C) 2023 AlliedModders LLC
+//
+// SourcePawn is free software: you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// SourcePawn is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+// FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// SourcePawn. If not, see http://www.gnu.org/licenses/.
+
+#pragma once
+
+#include <deque>
+
+#include "stl-allocator.h"
+
+namespace sp {
+namespace tr {
+
+template <typename T>
+using deque = std::deque<T, StlAllocator<T>>;
+
+} // namespace tr
+} // namespace sp
+