Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Python 3.12 support #10548

Merged
merged 2 commits into from
Aug 3, 2024
Merged

Python 3.12 support #10548

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1329b9d
3.12
jackalcooper Aug 1, 2024
36b96ae
port changes
jackalcooper Aug 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/release.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ jobs:
nightly-date: ${{ needs.generate-build-matrix.outputs.formatted_date }}
use-nvidia-wheels: ${{ matrix.entry !='cu112' }}
python-versions: |
3.12
3.11
3.10
3.9
Expand All @@ -195,6 +196,7 @@ jobs:
nightly-date: ${{ needs.generate-build-matrix.outputs.formatted_date }}
use-nvidia-wheels: ${{ matrix.entry !='cu112' }}
python-versions: |
3.12
3.11
3.10
3.9
Expand All @@ -219,6 +221,7 @@ jobs:
nightly: ${{ inputs.is_priv || github.event_name == 'schedule' || github.ref == 'refs/heads/release/add_nightly_date_index'}}
nightly-date: ${{ needs.generate-build-matrix.outputs.formatted_date }}
python-versions: |
3.12
3.11
3.10
3.9
Expand Down
3 changes: 2 additions & 1 deletion ci/fixed-dev-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
numpy==1.22.1 ; python_version >= "3.10"
numpy==1.26.4 ; python_version >= "3.12"
numpy==1.22.1 ; python_version >= "3.10" and python_version < "3.12"
numpy==1.21.6 ; python_version >= "3.7" and python_version < "3.10"
1 change: 1 addition & 0 deletions oneflow/api/python/autograd/autograd_function_state.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_AUTOGRAD_AUTOGRAD_FUNCTION_STATE_H_

#include <Python.h>
#undef _PyGC_FINALIZED
#include <pybind11/pybind11.h>

#include "oneflow/core/framework/op_expr_grad_function.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/caster/size.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_CASTER_SIZE_H_
#include <type_traits>
#include <Python.h>
#undef _PyGC_FINALIZED
#include <pybind11/pybind11.h>

#include "oneflow/api/python/framework/size.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/exception/exception.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_COMMON_EXCEPTION_H_

#include <Python.h>
#undef _PyGC_FINALIZED
#include <pybind11/pybind11.h>

#include "oneflow/core/common/exception.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/framework/memory_format.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FRAMEWORK_MEMORY_FORMAT_H_

#include <Python.h>
#undef _PyGC_FINALIZED
#include <pybind11/pybind11.h>

#include "oneflow/core/common/memory_format.pb.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/framework/size.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FRAMEWORK_SIZE_H_
#include <type_traits>
#include <Python.h>
#undef _PyGC_FINALIZED
#include <pybind11/pybind11.h>
#include "oneflow/core/common/shape.h"

Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/framework/tensor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.

#include <pybind11/pybind11.h>
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/api/python/exception/exception.h"
#include "oneflow/api/python/framework/size.h"
#include "oneflow/api/python/framework/tensortype.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/framework/tensor.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FRAMEWORK_TENSOR_H_

#include <Python.h>
#undef _PyGC_FINALIZED

#include "oneflow/core/framework/tensor.h"

Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/framework/tensor_functions.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ limitations under the License.
*/

#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/api/python/exception/exception.h"
#include "oneflow/api/python/framework/size.h"
#include "oneflow/api/python/framework/tensor_functions_util.h"
Expand Down
3 changes: 2 additions & 1 deletion oneflow/api/python/framework/tensor_functions_util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
#include <Python.h>
#undef _PyGC_FINALIZED
#include <string>
#include "oneflow/api/python/exception/exception.h"
#include "oneflow/api/python/functional/common.h"
Expand Down Expand Up @@ -71,4 +72,4 @@ PyObject* PyParseArgs(PyObject* args, PyObject* kwargs, const char* func_name,
}

} // namespace one
} // namespace oneflow
} // namespace oneflow
1 change: 1 addition & 0 deletions oneflow/api/python/framework/tensortype.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
#include <Python.h>
#undef _PyGC_FINALIZED
#include <pybind11/pybind11.h>
#include "oneflow/api/python/framework/tensor.h"
#include "oneflow/api/python/framework/tensortype.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/framework/tensortype.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FRAMEWORK_TENSORTYPE_H_

#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/core/framework/dtype.h"
#include "oneflow/core/framework/device.h"

Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/framework/typeinfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FRAMEWORK_TYPEINFO_H_

#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/core/common/throw.h"
#include "oneflow/core/framework/dtype.h"

Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/functional/indexing.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FUNCTIONAL_INDEXING_H_

#include <Python.h>
#undef _PyGC_FINALIZED

#include "oneflow/api/python/functional/common.h"
#include "oneflow/core/common/maybe.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/functional/python_arg.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ limitations under the License.

#include <pybind11/pybind11.h>
#include <Python.h>
#undef _PyGC_FINALIZED

#include "oneflow/core/common/throw.h"
#include "oneflow/api/python/functional/value_types.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/functional/python_arg_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FUNCTIONAL_PYTHON_ARG_PARSER_H_

#include <Python.h>
#undef _PyGC_FINALIZED

#include "oneflow/api/python/functional/function_def.h"
#include "oneflow/api/python/functional/python_arg.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/functional/python_return_types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FUNCTIONAL_PYTHON_RETURN_TYPES_H_

#include <Python.h>
#undef _PyGC_FINALIZED
#include <string>
#include <sstream>
#include <structmember.h>
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/functional/tensor_api.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
#include <Python.h>
#undef _PyGC_FINALIZED
#include <memory>

#include "oneflow/api/python/utils/tensor_utils.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/functional/value_types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ limitations under the License.
#include <complex>
#include <memory>
#include <Python.h>
#undef _PyGC_FINALIZED

#include "oneflow/core/common/data_type.pb.h"
#include "oneflow/core/common/maybe.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/utils/tensor_utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_UTILS_TENSOR_UTILS_H_

#include <Python.h>
#undef _PyGC_FINALIZED
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/functional.h>
Expand Down
1 change: 1 addition & 0 deletions oneflow/extension/python/py_compute.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ limitations under the License.
#ifndef ONEFLOW_EXTENSION_PYTHON_PY_COMPUTE_H_
#define ONEFLOW_EXTENSION_PYTHON_PY_COMPUTE_H_
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/core/framework/framework.h"

namespace oneflow {
Expand Down
1 change: 1 addition & 0 deletions oneflow/extension/python/py_kernel_registry.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ limitations under the License.

#include <string>
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/core/common/maybe.h"

namespace oneflow {
Expand Down
1 change: 1 addition & 0 deletions oneflow/extension/stack/python/custom_eval_frame.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ limitations under the License.

#define PY_SSIZE_T_CLEAN
#include <Python.h>
#undef _PyGC_FINALIZED
#include <frameobject.h>
#include <pystate.h>
// see https://bugs.python.org/issue35886
Expand Down
1 change: 1 addition & 0 deletions oneflow/extension/stack/python/custom_eval_frame.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ extern "C" {
#endif

#include <Python.h>
#undef _PyGC_FINALIZED

#if PY_VERSION_HEX >= 0x03090000
typedef PyObject* (*PyFrameEvalFunc)(struct _ts*, struct _frame*, int);
Expand Down
5 changes: 2 additions & 3 deletions python/oneflow/framework/check_point_v2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
IO,
BinaryIO,
)
from typing_extensions import TypeAlias
from pathlib import Path
import pickle
import json
Expand Down Expand Up @@ -58,10 +57,10 @@
PROTOCOL_VERSION = 1
ONEFLOW_MAGIC_KEY = "__oneflow__"

MAP_LOCATION: TypeAlias = Optional[
MAP_LOCATION = Optional[
Union[Callable[[Tensor, str], Tensor], flow.device, str, flow.placement]
]
FILE_LIKE: TypeAlias = Union[os.PathLike, BinaryIO, IO[bytes], Path]
FILE_LIKE = Union[os.PathLike, BinaryIO, IO[bytes], Path]


class _opener(object):
Expand Down
14 changes: 10 additions & 4 deletions python/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
See the License for the specific language governing permissions and
limitations under the License.
"""

from __future__ import absolute_import

import argparse
Expand Down Expand Up @@ -96,11 +97,16 @@ def has_ext_modules(self):


def get_oneflow_internal_so_path():
import imp

fp, pathname, description = imp.find_module("_oneflow_internal", ["oneflow"])
import importlib

suffixes = importlib.machinery.EXTENSION_SUFFIXES
loader = importlib.machinery.ExtensionFileLoader
lazy_loader = importlib.util.LazyLoader.factory(loader)
finder = importlib.machinery.FileFinder("oneflow", (lazy_loader, suffixes))
spec = finder.find_spec("_oneflow_internal")
pathname = spec.origin
assert os.path.isfile(pathname)
return os.path.relpath(pathname, "oneflow")
return os.path.basename(pathname)


package_data = {"oneflow": [get_oneflow_internal_so_path()] + include_files}
Expand Down
3 changes: 3 additions & 0 deletions tools/functional/generate_dispatch_stateful_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import re
import argparse
Expand Down Expand Up @@ -52,6 +53,7 @@
#define ONEFLOW_API_PYTHON_FUNCTIONAL_GENERATED_DISPATCH_OP_API_H_
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/core/common/optional.h"
#include "oneflow/core/common/scalar.h"
Expand Down Expand Up @@ -109,6 +111,7 @@
+ """
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/api/python/of_api_registry.h"
#include "oneflow/api/python/functional/common.h"
Expand Down
3 changes: 3 additions & 0 deletions tools/functional/generate_functional_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import re
import argparse
Expand Down Expand Up @@ -101,6 +102,7 @@
+ """
#include <Python.h>
#undef _PyGC_FINALIZED
namespace oneflow {{
namespace one {{
Expand All @@ -117,6 +119,7 @@
+ """
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/api/python/of_api_registry.h"
#include "oneflow/api/python/exception/exception.h"
Expand Down
4 changes: 4 additions & 0 deletions tools/functional/generate_tensor_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import re
import argparse
Expand Down Expand Up @@ -52,6 +53,7 @@
#define ONEFLOW_API_PYTHON_FUNCTIONAL_GENERATED_TENSOR_API_H_
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/core/common/optional.h"
#include "oneflow/core/common/scalar.h"
Expand Down Expand Up @@ -95,6 +97,7 @@
+ """
#include <Python.h>
#undef _PyGC_FINALIZED
namespace oneflow {{
namespace one {{
Expand All @@ -111,6 +114,7 @@
+ """
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/api/python/of_api_registry.h"
#include "oneflow/api/python/exception/exception.h"
Expand Down
Loading